From 1007938cb4b614d81df47b7935b9c5761c4dd5db Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Sun, 19 Apr 2026 10:02:55 +0000 Subject: [PATCH 1/7] feat(speech): add speech_service (ASR/TTS/Dialog) and audio_driver packages Add two new Robonix packages forming the voice interaction pipeline: - speech_service (robonix/srv/speech): gRPC service with ASR (Whisper + FunASR Paraformer), TTS (Edge TTS), and Dialog session management. Includes audio_utils for auto-adapting any input format to 16kHz mono pcm_s16le, eliminating the need for callers to pre-process audio. - audio_driver (robonix/prm/audio): ALSA auto-detection and streaming audio I/O via arecord/aplay subprocesses. Designed as a reference template for future prm driver packages (driver registry pattern, tiago_bridge startup flow, separate gRPC servers per interface). Also adds 7 contract TOMLs (2 prm audio + 5 srv speech), 19 ROS IDL message/service definitions (asr, tts, speech packages), generated proto files, and integrates both packages into the run.sh orchestration script with START_SPEECH_SERVICE / START_AUDIO_DRIVER env toggles. Smoke tested: Atlas registration, contract discovery via QueryNodes, and ASR/TTS RPCs all pass end-to-end. --- rust/contracts/prm/audio_mic.v1.toml | 16 + rust/contracts/prm/audio_speaker.v1.toml | 16 + rust/contracts/srv/speech_asr.v1.toml | 16 + rust/contracts/srv/speech_asr_stream.v1.toml | 19 + rust/contracts/srv/speech_dialog.v1.toml | 19 + rust/contracts/srv/speech_tts.v1.toml | 16 + rust/contracts/srv/speech_tts_stream.v1.toml | 18 + .../lib/asr/msg/AsrAudioChunk.msg | 2 + .../lib/asr/msg/RecognizeStreamEvent.msg | 12 + .../lib/asr/srv/Recognize.srv | 2 +- .../lib/asr/srv/RecognizeStream.srv | 15 +- .../lib/robonix_msg/msg/AudioChunk.msg | 8 + .../lib/robonix_msg/msg/AudioConfig.msg | 7 + .../lib/robonix_msg/msg/DialogEvent.msg | 34 + .../lib/robonix_msg/msg/DialogState.msg | 12 + .../lib/robonix_msg/msg/VadEvent.msg | 10 + .../lib/speech/msg/DialogRequest.msg | 7 + .../lib/speech/srv/GetDialogStatus.srv | 6 + .../lib/speech/srv/SendAudio.srv | 9 + .../lib/speech/srv/SendText.srv | 8 + .../lib/speech/srv/SetLanguage.srv | 7 + .../lib/speech/srv/StartDialog.srv | 10 + .../lib/speech/srv/StopDialog.srv | 6 + .../lib/tts/msg/SynthesizeAudioChunk.msg | 5 + .../lib/tts/srv/Synthesize.srv | 2 +- .../lib/tts/srv/SynthesizeStream.srv | 12 +- .../robonix_proto/asr.proto | 43 + .../robonix_proto/prm_base.proto | 59 -- .../robonix_proto/robonix_msg.proto | 77 +- .../robonix_proto/slam.proto | 29 - .../robonix_proto/speech.proto | 29 + .../robonix_proto/tts.proto | 41 + .../examples/packages/audio_driver/.gitignore | 5 + rust/examples/packages/audio_driver/README.md | 205 +++++ .../audio_driver/audio_driver/__init__.py | 15 + .../audio_driver/audio_driver/alsa_utils.py | 264 ++++++ .../audio_driver/audio_driver/mic_driver.py | 172 ++++ .../audio_driver/audio_driver/node.py | 405 +++++++++ .../audio_driver/speaker_driver.py | 154 ++++ .../audio_driver/proto/audio_driver.proto | 93 ++ .../packages/audio_driver/requirements.txt | 3 + .../audio_driver/robonix_manifest.yaml | 19 + .../packages/audio_driver/scripts/build.sh | 16 + .../packages/speech_service/.gitignore | 5 + .../packages/speech_service/README.md | 137 +++ .../speech_service/proto/speech_service.proto | 154 ++++ .../speech_service/rbnx_speech_smoke.py | 271 ++++++ .../packages/speech_service/requirements.txt | 11 + .../speech_service/robonix_manifest.yaml | 23 + .../packages/speech_service/scripts/build.sh | 25 + .../speech_service/speech_service/__init__.py | 11 + .../speech_service/audio_utils.py | 221 +++++ .../speech_service/speech_service/service.py | 841 ++++++++++++++++++ rust/examples/run.sh | 20 +- 54 files changed, 3499 insertions(+), 143 deletions(-) create mode 100644 rust/contracts/prm/audio_mic.v1.toml create mode 100644 rust/contracts/prm/audio_speaker.v1.toml create mode 100644 rust/contracts/srv/speech_asr.v1.toml create mode 100644 rust/contracts/srv/speech_asr_stream.v1.toml create mode 100644 rust/contracts/srv/speech_dialog.v1.toml create mode 100644 rust/contracts/srv/speech_tts.v1.toml create mode 100644 rust/contracts/srv/speech_tts_stream.v1.toml create mode 100644 rust/crates/robonix-interfaces/lib/asr/msg/AsrAudioChunk.msg create mode 100644 rust/crates/robonix-interfaces/lib/asr/msg/RecognizeStreamEvent.msg create mode 100644 rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioChunk.msg create mode 100644 rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioConfig.msg create mode 100644 rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogEvent.msg create mode 100644 rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogState.msg create mode 100644 rust/crates/robonix-interfaces/lib/robonix_msg/msg/VadEvent.msg create mode 100644 rust/crates/robonix-interfaces/lib/speech/msg/DialogRequest.msg create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/GetDialogStatus.srv create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/SendAudio.srv create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/SendText.srv create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/SetLanguage.srv create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv create mode 100644 rust/crates/robonix-interfaces/lib/speech/srv/StopDialog.srv create mode 100644 rust/crates/robonix-interfaces/lib/tts/msg/SynthesizeAudioChunk.msg create mode 100644 rust/crates/robonix-interfaces/robonix_proto/asr.proto create mode 100644 rust/crates/robonix-interfaces/robonix_proto/speech.proto create mode 100644 rust/crates/robonix-interfaces/robonix_proto/tts.proto create mode 100644 rust/examples/packages/audio_driver/.gitignore create mode 100644 rust/examples/packages/audio_driver/README.md create mode 100644 rust/examples/packages/audio_driver/audio_driver/__init__.py create mode 100644 rust/examples/packages/audio_driver/audio_driver/alsa_utils.py create mode 100644 rust/examples/packages/audio_driver/audio_driver/mic_driver.py create mode 100644 rust/examples/packages/audio_driver/audio_driver/node.py create mode 100644 rust/examples/packages/audio_driver/audio_driver/speaker_driver.py create mode 100644 rust/examples/packages/audio_driver/proto/audio_driver.proto create mode 100644 rust/examples/packages/audio_driver/requirements.txt create mode 100644 rust/examples/packages/audio_driver/robonix_manifest.yaml create mode 100644 rust/examples/packages/audio_driver/scripts/build.sh create mode 100644 rust/examples/packages/speech_service/.gitignore create mode 100644 rust/examples/packages/speech_service/README.md create mode 100644 rust/examples/packages/speech_service/proto/speech_service.proto create mode 100644 rust/examples/packages/speech_service/rbnx_speech_smoke.py create mode 100644 rust/examples/packages/speech_service/requirements.txt create mode 100644 rust/examples/packages/speech_service/robonix_manifest.yaml create mode 100755 rust/examples/packages/speech_service/scripts/build.sh create mode 100644 rust/examples/packages/speech_service/speech_service/__init__.py create mode 100644 rust/examples/packages/speech_service/speech_service/audio_utils.py create mode 100644 rust/examples/packages/speech_service/speech_service/service.py diff --git a/rust/contracts/prm/audio_mic.v1.toml b/rust/contracts/prm/audio_mic.v1.toml new file mode 100644 index 000000000..203de5f42 --- /dev/null +++ b/rust/contracts/prm/audio_mic.v1.toml @@ -0,0 +1,16 @@ +# Microphone audio stream (raw PCM or encoded). +# ROS: `robonix_msg/msg/AudioChunk` published on a topic. +[contract] +id = "robonix/prm/audio/mic" +version = "1" +kind = "primitive" + +[io.msg] +msg = "robonix_msg/msg/AudioChunk" + +[mode] +type = "topic_out" + +[semantics] +realtime = true +payload = "audio" diff --git a/rust/contracts/prm/audio_speaker.v1.toml b/rust/contracts/prm/audio_speaker.v1.toml new file mode 100644 index 000000000..cd225dc40 --- /dev/null +++ b/rust/contracts/prm/audio_speaker.v1.toml @@ -0,0 +1,16 @@ +# Speaker audio stream input (raw PCM or encoded). +# ROS: `robonix_msg/msg/AudioChunk` consumed from a topic. +[contract] +id = "robonix/prm/audio/speaker" +version = "1" +kind = "primitive" + +[io.msg] +msg = "robonix_msg/msg/AudioChunk" + +[mode] +type = "topic_in" + +[semantics] +realtime = true +payload = "audio" diff --git a/rust/contracts/srv/speech_asr.v1.toml b/rust/contracts/srv/speech_asr.v1.toml new file mode 100644 index 000000000..a3500b380 --- /dev/null +++ b/rust/contracts/srv/speech_asr.v1.toml @@ -0,0 +1,16 @@ +# ASR one-shot recognition. +# Caller sends raw audio; returns transcript. +# ROS: `lib/asr/srv/Recognize.srv`. +[contract] +id = "robonix/srv/speech/asr" +version = "1" +kind = "service" + +[io.srv] +srv = "asr/srv/Recognize" + +[mode] +type = "rpc" + +[semantics] +stateless = true diff --git a/rust/contracts/srv/speech_asr_stream.v1.toml b/rust/contracts/srv/speech_asr_stream.v1.toml new file mode 100644 index 000000000..8b3dcf128 --- /dev/null +++ b/rust/contracts/srv/speech_asr_stream.v1.toml @@ -0,0 +1,19 @@ +# ASR streaming recognition. +# Client streams AudioChunk; server streams RecognizeStreamEvent. +# ROS: `lib/asr/srv/RecognizeStream.srv`. Stream element types in TOML only. +[contract] +id = "robonix/srv/speech/asr_stream" +version = "1" +kind = "service" + +[io.srv] +srv = "asr/srv/RecognizeStream" +stream_request = "asr/msg/AsrAudioChunk" +stream_response = "asr/msg/RecognizeStreamEvent" + +[mode] +type = "rpc_bidirectional_stream" + +[semantics] +stateless = true +realtime = true diff --git a/rust/contracts/srv/speech_dialog.v1.toml b/rust/contracts/srv/speech_dialog.v1.toml new file mode 100644 index 000000000..4a7968a24 --- /dev/null +++ b/rust/contracts/srv/speech_dialog.v1.toml @@ -0,0 +1,19 @@ +# Voice dialog session manager. +# Orchestrates mic → ASR → intent → TTS → speaker pipeline. +# Manages VAD, dialog state machine, and language switching. +# ROS: `lib/speech/srv/StartDialog.srv`. Stream element: `robonix_msg/msg/DialogEvent`. +[contract] +id = "robonix/srv/speech/dialog" +version = "1" +kind = "service" + +[io.srv] +srv = "speech/srv/StartDialog" +stream_response = "robonix_msg/msg/DialogEvent" + +[mode] +type = "rpc_server_stream" + +[semantics] +stateful = true +interruptible = true diff --git a/rust/contracts/srv/speech_tts.v1.toml b/rust/contracts/srv/speech_tts.v1.toml new file mode 100644 index 000000000..8eadaea4a --- /dev/null +++ b/rust/contracts/srv/speech_tts.v1.toml @@ -0,0 +1,16 @@ +# TTS one-shot synthesis. +# Caller sends text; returns synthesized audio. +# ROS: `lib/tts/srv/Synthesize.srv`. +[contract] +id = "robonix/srv/speech/tts" +version = "1" +kind = "service" + +[io.srv] +srv = "tts/srv/Synthesize" + +[mode] +type = "rpc" + +[semantics] +stateless = true diff --git a/rust/contracts/srv/speech_tts_stream.v1.toml b/rust/contracts/srv/speech_tts_stream.v1.toml new file mode 100644 index 000000000..9deb61e17 --- /dev/null +++ b/rust/contracts/srv/speech_tts_stream.v1.toml @@ -0,0 +1,18 @@ +# TTS streaming synthesis. +# Client sends one SynthesizeStreamRequest; server streams back SynthesizeAudioChunk. +# ROS: `lib/tts/srv/SynthesizeStream.srv`. Stream element type in TOML only. +[contract] +id = "robonix/srv/speech/tts_stream" +version = "1" +kind = "service" + +[io.srv] +srv = "tts/srv/SynthesizeStream" +stream_response = "tts/msg/SynthesizeAudioChunk" + +[mode] +type = "rpc_server_stream" + +[semantics] +stateless = true +realtime = true diff --git a/rust/crates/robonix-interfaces/lib/asr/msg/AsrAudioChunk.msg b/rust/crates/robonix-interfaces/lib/asr/msg/AsrAudioChunk.msg new file mode 100644 index 000000000..5f0562353 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/asr/msg/AsrAudioChunk.msg @@ -0,0 +1,2 @@ +# Audio chunk sent by client during streaming ASR recognition. +robonix_msg/msg/AudioChunk chunk diff --git a/rust/crates/robonix-interfaces/lib/asr/msg/RecognizeStreamEvent.msg b/rust/crates/robonix-interfaces/lib/asr/msg/RecognizeStreamEvent.msg new file mode 100644 index 000000000..8733150f8 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/asr/msg/RecognizeStreamEvent.msg @@ -0,0 +1,12 @@ +# Event streamed back by the ASR service during streaming recognition. + +uint8 PARTIAL = 0 +uint8 FINAL = 1 +uint8 ERROR = 2 +uint8 event_type + +string text # recognised text so far (partial or final) +float32 confidence # 0.0 – 1.0 +string language # detected language (BCP-47) +bool is_final # true when this is the final recognition result +string error # non-empty on error diff --git a/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv b/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv index d8884aa72..17b8adedc 100644 --- a/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv +++ b/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv @@ -2,7 +2,7 @@ # abstract_interface_id: robonix/sys/speech/asr/recognize # Caller sends raw audio bytes; service returns a transcript string. --- -bytes audio_data +uint8[] audio_data string encoding # e.g. "pcm_s16le", "opus", "mp3" uint32 sample_rate_hz # e.g. 16000 string language # BCP-47, e.g. "zh-CN", "en-US", "" = auto-detect diff --git a/rust/crates/robonix-interfaces/lib/asr/srv/RecognizeStream.srv b/rust/crates/robonix-interfaces/lib/asr/srv/RecognizeStream.srv index 5e4831233..fbe5a7d8d 100644 --- a/rust/crates/robonix-interfaces/lib/asr/srv/RecognizeStream.srv +++ b/rust/crates/robonix-interfaces/lib/asr/srv/RecognizeStream.srv @@ -1,10 +1,7 @@ -# ASR streaming recognition -# @robonix.grpc stream_client stream_server -# abstract_interface_id: robonix/sys/speech/asr/recognize_stream -# -# Client streams AudioChunk messages; server streams back RecognizeStreamEvent. -# Use for real-time microphone input. +# ASR streaming recognition (bidirectional stream). +# Client streams AsrAudioChunk; server streams back RecognizeStreamEvent. +# Stream element types are referenced in the contract TOML. +robonix_msg/AudioConfig audio_config +string language --- -# (empty — streaming RPC, no single request body) -=== -# (empty — streaming RPC, no single response body) +asr/RecognizeStreamEvent event diff --git a/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioChunk.msg b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioChunk.msg new file mode 100644 index 000000000..85216c4b8 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioChunk.msg @@ -0,0 +1,8 @@ +# A chunk of raw or encoded audio data. +# Used as the stream element for mic topic_out, speaker topic_in, +# ASR streaming, and TTS streaming. + +uint64 timestamp_ns # nanoseconds since epoch (replaces std_msgs/Header for portability) +uint8[] data # raw or encoded audio bytes +uint32 sequence # monotonically increasing within a session +float32 duration_s # duration of this chunk in seconds (0 if unknown) diff --git a/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioConfig.msg b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioConfig.msg new file mode 100644 index 000000000..77f0c67c5 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/AudioConfig.msg @@ -0,0 +1,7 @@ +# Audio configuration descriptor for speech services. +# Shared across ASR, TTS, mic, and speaker primitives. + +string encoding # e.g. "pcm_s16le", "pcm_f32le", "opus", "mp3" +uint32 sample_rate_hz # e.g. 16000, 22050, 44100 +uint32 channels # 1 = mono, 2 = stereo +uint32 bits_per_sample # e.g. 16 for pcm_s16le diff --git a/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogEvent.msg b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogEvent.msg new file mode 100644 index 000000000..f2b627b9c --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogEvent.msg @@ -0,0 +1,34 @@ +# Event emitted by the speech dialog service during a voice interaction session. +# Streamed back to the caller (e.g. Liaison) over the dialog RPC stream. + +uint64 timestamp_ns # nanoseconds since epoch +string session_id + +uint8 VAD_EVENT = 0 +uint8 PARTIAL_TRANSCRIPT = 1 +uint8 FINAL_TRANSCRIPT = 2 +uint8 BOT_RESPONSE_TEXT = 3 +uint8 BOT_AUDIO_CHUNK = 4 +uint8 STATE_CHANGE = 5 +uint8 ERROR = 6 +uint8 event_type + +# VAD event data +robonix_msg/msg/VadEvent vad_event + +# Transcript data (for PARTIAL_TRANSCRIPT and FINAL_TRANSCRIPT) +string text +float32 confidence +string language # detected or specified language + +# Bot response text (for BOT_RESPONSE_TEXT) +string response_text + +# Audio data (for BOT_AUDIO_CHUNK) +robonix_msg/msg/AudioChunk audio_chunk + +# State change (for STATE_CHANGE) +robonix_msg/msg/DialogState dialog_state + +# Error info (for ERROR) +string error diff --git a/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogState.msg b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogState.msg new file mode 100644 index 000000000..2b64ea7da --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/DialogState.msg @@ -0,0 +1,12 @@ +# State of a voice dialog session. +# Used to track the lifecycle of a voice interaction. + +string session_id +uint8 IDLE = 0 +uint8 LISTENING = 1 # mic is active, waiting for speech +uint8 PROCESSING = 2 # ASR/NLP processing in progress +uint8 SPEAKING = 3 # TTS is playing audio to speaker +uint8 ERROR = 4 +uint8 state +string language # current BCP-47 language code +string error # non-empty if state == ERROR diff --git a/rust/crates/robonix-interfaces/lib/robonix_msg/msg/VadEvent.msg b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/VadEvent.msg new file mode 100644 index 000000000..20ec59265 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/robonix_msg/msg/VadEvent.msg @@ -0,0 +1,10 @@ +# Voice Activity Detection event. +# Emitted by the dialog manager or ASR service to signal speech state changes. + +uint64 timestamp_ns # nanoseconds since epoch +uint8 SPEECH_BEGIN = 0 +uint8 SPEECH_END = 1 +uint8 SILENCE = 2 +uint8 event_type +float32 confidence # 0.0 – 1.0 confidence of the VAD decision +float32 energy_level # audio energy level (optional, 0.0 if unavailable) diff --git a/rust/crates/robonix-interfaces/lib/speech/msg/DialogRequest.msg b/rust/crates/robonix-interfaces/lib/speech/msg/DialogRequest.msg new file mode 100644 index 000000000..347d955eb --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/msg/DialogRequest.msg @@ -0,0 +1,7 @@ +# Dialog request sent by client to start a voice dialog session. +string language # BCP-47, e.g. "zh-CN", "" = auto-detect +string voice_id # preferred voice, "" = default +bool enable_vad # enable voice activity detection +float32 vad_silence_threshold_s # seconds of silence before ending speech (default 0.8) +float32 vad_energy_threshold # energy threshold for VAD (0.0 = auto) +robonix_msg/msg/AudioConfig audio_config # desired audio format diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/GetDialogStatus.srv b/rust/crates/robonix-interfaces/lib/speech/srv/GetDialogStatus.srv new file mode 100644 index 000000000..6d7b2007e --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/GetDialogStatus.srv @@ -0,0 +1,6 @@ +# Query current dialog session status. +--- +string session_id +--- +robonix_msg/msg/DialogState state +string error diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/SendAudio.srv b/rust/crates/robonix-interfaces/lib/speech/srv/SendAudio.srv new file mode 100644 index 000000000..063b49a90 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/SendAudio.srv @@ -0,0 +1,9 @@ +# Inject audio into an active dialog session (for non-streaming input). +--- +string session_id +uint8[] audio_data +string encoding +uint32 sample_rate_hz +--- +bool accepted +string error diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/SendText.srv b/rust/crates/robonix-interfaces/lib/speech/srv/SendText.srv new file mode 100644 index 000000000..3b8c1c232 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/SendText.srv @@ -0,0 +1,8 @@ +# Send a text-based message into the dialog (bypass ASR). +--- +string session_id +string text +string language +--- +bool accepted +string error diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/SetLanguage.srv b/rust/crates/robonix-interfaces/lib/speech/srv/SetLanguage.srv new file mode 100644 index 000000000..e268a5715 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/SetLanguage.srv @@ -0,0 +1,7 @@ +# Change language mid-session. +--- +string session_id +string language # BCP-47 code +--- +bool success +string error diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv b/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv new file mode 100644 index 000000000..713fd1729 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv @@ -0,0 +1,10 @@ +# Start a voice dialog session. +# Returns initial dialog state; events are streamed via the Dialog contract. +--- +robonix_msg/msg/AudioConfig audio_config +string language +bool enable_vad +--- +string session_id +robonix_msg/msg/DialogState state +string error diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/StopDialog.srv b/rust/crates/robonix-interfaces/lib/speech/srv/StopDialog.srv new file mode 100644 index 000000000..db7b992b5 --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/speech/srv/StopDialog.srv @@ -0,0 +1,6 @@ +# Stop an active voice dialog session. +--- +string session_id +--- +bool success +string error diff --git a/rust/crates/robonix-interfaces/lib/tts/msg/SynthesizeAudioChunk.msg b/rust/crates/robonix-interfaces/lib/tts/msg/SynthesizeAudioChunk.msg new file mode 100644 index 000000000..eb02cd59a --- /dev/null +++ b/rust/crates/robonix-interfaces/lib/tts/msg/SynthesizeAudioChunk.msg @@ -0,0 +1,5 @@ +# Audio chunk streamed back by TTS during streaming synthesis. +robonix_msg/msg/AudioChunk chunk +string encoding # e.g. "pcm_s16le", "mp3" +uint32 sample_rate_hz +bool is_final # true when this is the last chunk diff --git a/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv b/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv index 6f13eb084..ab4c1623c 100644 --- a/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv +++ b/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv @@ -6,7 +6,7 @@ string language # BCP-47, e.g. "zh-CN", "" = default string voice # voice ID, "" = default float32 speed # 1.0 = normal; < 1.0 = slower; > 1.0 = faster === -bytes audio_data # synthesized audio (format specified in encoding) +uint8[] audio_data # synthesized audio (format specified in encoding) string encoding # e.g. "pcm_s16le", "mp3" uint32 sample_rate_hz string error # non-empty on failure diff --git a/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv b/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv index a02372519..3fffe4f3c 100644 --- a/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv +++ b/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv @@ -1,13 +1,11 @@ -# TTS streaming synthesis -# @robonix.grpc stream_server -# abstract_interface_id: robonix/sys/speech/tts/synthesize_stream -# -# Client sends one SynthesizeStreamRequest; server streams back AudioChunk. -# Use for low-latency playback while text is still being generated. +# TTS streaming synthesis (server stream). +# Client sends text request; server streams back SynthesizeAudioChunk. +# Stream element type is referenced in the contract TOML. --- string text string language string voice float32 speed +robonix_msg/AudioConfig audio_config === -# (empty — streaming RPC, no single response body) +tts/SynthesizeAudioChunk chunk diff --git a/rust/crates/robonix-interfaces/robonix_proto/asr.proto b/rust/crates/robonix-interfaces/robonix_proto/asr.proto new file mode 100644 index 000000000..4e3ad2827 --- /dev/null +++ b/rust/crates/robonix-interfaces/robonix_proto/asr.proto @@ -0,0 +1,43 @@ +// @generated by robonix-codegen --lang proto +// source: ROS IDL package 'asr' +syntax = "proto3"; + +package robonix.asr; + +import "robonix_msg.proto"; + +message AsrAudioChunk { + robonix.robonix_msg.AudioChunk chunk = 1; +} + +message RecognizeStreamEvent { + uint32 event_type = 1; + string text = 2; + float confidence = 3; + string language = 4; + bool is_final = 5; + string error = 6; +} + +message Recognize_Request { +} + +message Recognize_Response { + bytes audio_data = 1; + string encoding = 2; + uint32 sample_rate_hz = 3; + string language = 4; + string text = 5; + float confidence = 6; + string error = 7; +} + +message RecognizeStream_Request { + robonix.robonix_msg.AudioConfig audio_config = 1; + string language = 2; +} + +message RecognizeStream_Response { + RecognizeStreamEvent event = 1; +} + diff --git a/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto b/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto index f13e7c8ee..19fc74a12 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto @@ -4,10 +4,6 @@ syntax = "proto3"; package robonix.prm_base; -import "geometry_msgs.proto"; -import "sensor_msgs.proto"; -import "std_msgs.proto"; - message MoveCommand { double linear_x = 1; double linear_y = 2; @@ -17,15 +13,6 @@ message MoveCommand { double angular_z = 6; } -message RobotState { - robonix.std_msgs.Header header = 1; - robonix.geometry_msgs.PoseStamped base_pose = 2; - robonix.sensor_msgs.JointState joint_state = 3; - robonix.geometry_msgs.PoseStamped tcp_pose = 4; - double gripper_width = 5; - bool is_grasped = 6; -} - message CancelNavigation_Request { string goal_id = 1; } @@ -35,28 +22,6 @@ message CancelNavigation_Response { string message = 2; } -message ExecuteMoveCommand_Request { - MoveCommand command = 1; -} - -message ExecuteMoveCommand_Response { - robonix.std_msgs.String status = 1; -} - -message GetCameraImage_Request { -} - -message GetCameraImage_Response { - robonix.sensor_msgs.Image image = 1; -} - -message GetLaserScan_Request { -} - -message GetLaserScan_Response { - robonix.sensor_msgs.LaserScan scan = 1; -} - message GetNavigationStatus_Request { string goal_id = 1; } @@ -67,30 +32,6 @@ message GetNavigationStatus_Response { bool terminal = 3; } -message GetRobotState_Request { -} - -message GetRobotState_Response { - RobotState state = 1; -} - -message Navigate_Request { - robonix.geometry_msgs.PoseStamped goal = 1; -} - -message Navigate_Response { - bool accepted = 1; - string status_message = 2; -} - -message PerceptionDetect_Request { - robonix.sensor_msgs.Image image = 1; -} - -message PerceptionDetect_Response { - robonix.std_msgs.String results = 1; -} - message StreamMove_Request { MoveCommand cmd = 1; } diff --git a/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto b/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto index c6d4716c3..1c6959a57 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto @@ -4,9 +4,19 @@ syntax = "proto3"; package robonix.robonix_msg; -import "geometry_msgs.proto"; -import "sensor_msgs.proto"; -import "std_msgs.proto"; +message AudioChunk { + uint64 timestamp_ns = 1; + bytes data = 2; + uint32 sequence = 3; + float duration_s = 4; +} + +message AudioConfig { + string encoding = 1; + uint32 sample_rate_hz = 2; + uint32 channels = 3; + uint32 bits_per_sample = 4; +} message BoundingBox { double scale_x = 1; @@ -62,6 +72,27 @@ message CommandResult { int32 code = 3; } +message DialogEvent { + uint64 timestamp_ns = 1; + string session_id = 2; + uint32 event_type = 3; + VadEvent vad_event = 4; + string text = 5; + float confidence = 6; + string language = 7; + string response_text = 8; + AudioChunk audio_chunk = 9; + DialogState dialog_state = 10; + string error = 11; +} + +message DialogState { + string session_id = 1; + uint32 state = 2; + string language = 3; + string error = 4; +} + message Dict { repeated string keys = 1; repeated string values = 2; @@ -92,16 +123,6 @@ message Object { repeated FrameMapping frame_mapping = 6; } -message Occupancy3D { - robonix.std_msgs.Header header = 1; - uint32 width = 2; - uint32 height = 3; - uint32 depth = 4; - float resolution = 5; - robonix.geometry_msgs.Point origin = 6; - repeated int32 data = 7; -} - message Point3D { double x = 1; double y = 2; @@ -116,11 +137,6 @@ message PrimitiveInstance { string metadata = 5; } -message RGBD { - robonix.sensor_msgs.Image rgb = 1; - robonix.sensor_msgs.Image depth = 2; -} - message Relation { RelationType relation_type = 1; string target_entity_id = 2; @@ -175,6 +191,13 @@ message ToolSpec { string input_schema_json = 3; } +message VadEvent { + uint64 timestamp_ns = 1; + uint32 event_type = 2; + float confidence = 3; + float energy_level = 4; +} + message Waypoint { string name = 1; double x = 2; @@ -183,21 +206,3 @@ message Waypoint { string frame_id = 5; } -message ZeroCopyFrame { - robonix.std_msgs.Header header = 1; - uint64 buffer_handle_id = 2; - string shm_name = 3; - uint32 width = 4; - uint32 height = 5; - uint32 channels = 6; - uint32 stride = 7; - BufferFormat format = 8; - MemoryDomain memory_domain = 9; - uint64 data_size = 10; - uint64 sequence = 11; - uint64 hw_timestamp_ns = 12; - uint64 sync_token = 13; - bool gpu_pinned = 14; - uint32 pool_id = 15; -} - diff --git a/rust/crates/robonix-interfaces/robonix_proto/slam.proto b/rust/crates/robonix-interfaces/robonix_proto/slam.proto index 49a21af68..1b2695815 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/slam.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/slam.proto @@ -4,26 +4,6 @@ syntax = "proto3"; package robonix.slam; -import "geometry_msgs.proto"; -import "std_msgs.proto"; - -message SlamStatus { - robonix.std_msgs.Header header = 1; - string mode = 2; - bool odom_alive = 3; - float odom_hz = 4; - bool cloud_alive = 5; - string map_file = 6; - uint64 total_odom_frames = 7; -} - -message GetSlamStatus_Request { -} - -message GetSlamStatus_Response { - SlamStatus status = 1; -} - message LoadMap_Request { string path = 1; } @@ -43,15 +23,6 @@ message SaveMap_Response { string message = 3; } -message SetInitialPose_Request { - robonix.geometry_msgs.PoseStamped pose = 1; -} - -message SetInitialPose_Response { - bool success = 1; - string message = 2; -} - message SwitchMode_Request { string mode = 1; } diff --git a/rust/crates/robonix-interfaces/robonix_proto/speech.proto b/rust/crates/robonix-interfaces/robonix_proto/speech.proto new file mode 100644 index 000000000..4d49272aa --- /dev/null +++ b/rust/crates/robonix-interfaces/robonix_proto/speech.proto @@ -0,0 +1,29 @@ +// @generated by robonix-codegen --lang proto +// source: ROS IDL package 'speech' +syntax = "proto3"; + +package robonix.speech; + +import "robonix_msg.proto"; + +message DialogRequest { + string language = 1; + string voice_id = 2; + bool enable_vad = 3; + float vad_silence_threshold_s = 4; + float vad_energy_threshold = 5; + robonix.robonix_msg.AudioConfig audio_config = 6; +} + +message StartDialog_Request { +} + +message StartDialog_Response { + robonix.robonix_msg.AudioConfig audio_config = 1; + string language = 2; + bool enable_vad = 3; + string session_id = 4; + robonix.robonix_msg.DialogState state = 5; + string error = 6; +} + diff --git a/rust/crates/robonix-interfaces/robonix_proto/tts.proto b/rust/crates/robonix-interfaces/robonix_proto/tts.proto new file mode 100644 index 000000000..5311a6914 --- /dev/null +++ b/rust/crates/robonix-interfaces/robonix_proto/tts.proto @@ -0,0 +1,41 @@ +// @generated by robonix-codegen --lang proto +// source: ROS IDL package 'tts' +syntax = "proto3"; + +package robonix.tts; + +import "robonix_msg.proto"; + +message SynthesizeAudioChunk { + robonix.robonix_msg.AudioChunk chunk = 1; + string encoding = 2; + uint32 sample_rate_hz = 3; + bool is_final = 4; +} + +message Synthesize_Request { +} + +message Synthesize_Response { + string text = 1; + string language = 2; + string voice = 3; + float speed = 4; + bytes audio_data = 5; + string encoding = 6; + uint32 sample_rate_hz = 7; + string error = 8; +} + +message SynthesizeStream_Request { +} + +message SynthesizeStream_Response { + string text = 1; + string language = 2; + string voice = 3; + float speed = 4; + robonix.robonix_msg.AudioConfig audio_config = 5; + SynthesizeAudioChunk chunk = 6; +} + diff --git a/rust/examples/packages/audio_driver/.gitignore b/rust/examples/packages/audio_driver/.gitignore new file mode 100644 index 000000000..a10042696 --- /dev/null +++ b/rust/examples/packages/audio_driver/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +proto_gen/ +rbnx-build/ +*.egg-info/ diff --git a/rust/examples/packages/audio_driver/README.md b/rust/examples/packages/audio_driver/README.md new file mode 100644 index 000000000..9c6796f09 --- /dev/null +++ b/rust/examples/packages/audio_driver/README.md @@ -0,0 +1,205 @@ +# Audio Driver + +音频驱动 — Robonix prm 层，自动扫描 ALSA 设备，提供麦克风采集和扬声器播放的 gRPC 流式接口。 + +**本包同时作为所有未来 prm driver 包的参考模板。** + +## 架构定位 + +``` +┌─────────────────────────────────────────────┐ +│ speech_service (robonix/srv/speech) │ +│ ↕ gRPC: AudioChunk │ +├─────────────────────────────────────────────┤ +│ audio_driver ← 你在这里 (robonix/prm/audio) │ +│ ↕ ALSA: arecord / aplay │ +├─────────────────────────────────────────────┤ +│ Hardware (USB Mic / Speaker) │ +└─────────────────────────────────────────────┘ +``` + +- **上层**：通过 gRPC 向 speech_service 等服务提供音频流 +- **下层**：通过 ALSA 子系统直接操控声卡硬件 +- **自动化**：启动时自动扫描 USB 麦克风/扬声器，无需手动配置 + +## gRPC 接口 + +| 服务 | RPC | 模式 | Contract ID | 数据流 | +|------|-----|------|-------------|--------| +| PrmAudioMic | Stream | Server-stream | `robonix/prm/audio/mic` | 麦克风 → 调用方 | +| PrmAudioSpeaker | Stream | Client-stream | `robonix/prm/audio/speaker` | 调用方 → 扬声器 | + +## 目录结构 + +``` +audio_driver/ +├── proto/ +│ └── audio_driver.proto # 自包含 gRPC 定义 +├── proto_gen/ # build 时生成的 *_pb2.py（git 忽略） +├── scripts/ +│ └── build.sh # proto 代码生成 +├── audio_driver/ +│ ├── __init__.py # 包入口 +│ ├── node.py # 主入口：Atlas 注册 + daemon 线程 + main() +│ ├── alsa_utils.py # ALSA 设备扫描 + 驱动注册表 +│ ├── mic_driver.py # 麦克风采集（arecord subprocess） +│ └── speaker_driver.py # 扬声器播放（aplay subprocess） +├── robonix_manifest.yaml # Robonix 包描述 +├── requirements.txt +└── .gitignore +``` + +## 快速启动 + +### 1. 安装依赖 + +```bash +pip install -r requirements.txt +# 核心依赖：grpcio, grpcio-tools, protobuf +# 系统依赖：arecord, aplay（alsa-utils 包） +``` + +### 2. 生成 proto stubs + +```bash +bash scripts/build.sh +``` + +### 3. 启动驱动 + +```bash +# 正常模式（自动扫描设备 + Atlas 注册） +export PYTHONPATH=$(pwd)/proto_gen:${PYTHONPATH:-} +python -m audio_driver.node + +# 独立模式（跳过 Atlas 注册） +AUDIO_DRIVER_STANDALONE=1 python -m audio_driver.node + +# 指定设备（覆盖自动检测） +AUDIO_MIC_DEVICE=hw:1,0 AUDIO_SPEAKER_DEVICE=hw:0,0 python -m audio_driver.node +``` + +### 4. 通过 Robonix 启动 + +```bash +rbnx run com.robonix.example.audio_driver +``` + +## 环境变量 + +| 变量 | 默认值 | 说明 | +|------|--------|------| +| `AUDIO_MIC_DEVICE` | 自动检测 | 麦克风 ALSA 设备（如 `hw:1,0`） | +| `AUDIO_MIC_SAMPLE_RATE` | `16000` | 采集采样率（Hz） | +| `AUDIO_MIC_CHANNELS` | `1` | 采集声道数 | +| `AUDIO_MIC_BITS` | `16` | 采集位深 | +| `AUDIO_MIC_CHUNK_MS` | `100` | 每次 chunk 时长（毫秒） | +| `AUDIO_MIC_PORT` | `0`（自动分配） | Mic gRPC 端口 | +| `AUDIO_SPEAKER_DEVICE` | 自动检测 | 扬声器 ALSA 设备 | +| `AUDIO_SPEAKER_SAMPLE_RATE` | `24000` | 播放采样率（Hz） | +| `AUDIO_SPEAKER_CHANNELS` | `1` | 播放声道数 | +| `AUDIO_SPEAKER_BITS` | `16` | 播放位深 | +| `AUDIO_SPEAKER_PORT` | `0`（自动分配） | Speaker gRPC 端口 | +| `AUDIO_DRIVER_STANDALONE` | — | 设为 `1` 跳过 Atlas 注册 | +| `ROBONIX_ATLAS` | `localhost:50051` | Atlas 控制面地址 | +| `ROBONIX_NODE_ID` | `com.robonix.prm.audio` | Atlas 节点 ID | + +## 设备自动发现 + +启动时执行以下流程： + +``` +1. arecord -l → 解析输入设备（麦克风） +2. aplay -l → 解析输出设备（扬声器） +3. 合并去重 → 同一 card/device 的设备标记 is_input + is_output +4. 选择默认 → 优先 USB 设备（card >= 1），其次任意可用设备 +5. 环境变量覆盖 → 如果设置了 AUDIO_MIC_DEVICE / AUDIO_SPEAKER_DEVICE，使用指定值 +``` + +## 驱动注册表 + +支持通过插件机制添加硬件专用驱动： + +```python +from audio_driver.alsa_utils import AudioDeviceDriver, register_driver + +class RespeakerDriver(AudioDeviceDriver): + """ReSpeaker 多通道麦克风阵列驱动""" + + def detect(self, devices): + return [d for d in devices if "ReSpeaker" in d.name] + + def name(self): + return "ReSpeaker Driver" + +register_driver(RespeakerDriver()) +``` + +内置驱动： +- **DefaultAlsaDriver** — 接受所有标准 ALSA 设备（默认注册） + +## 模块说明 + +### mic_driver.py — 麦克风采集 + +``` +ALSA 设备 → arecord subprocess → stdout → MicDriver.read_chunk() → AudioChunk dict +``` + +- 调用 `arecord -D hw:X,Y -f S16_LE -r 16000 -c 1 -t raw` +- `read_chunk()` 阻塞读取固定大小字节（默认 100ms = 3200 bytes @ 16kHz mono s16le） +- 返回 dict: `{timestamp_ns, data, sequence, duration_s}` + +### speaker_driver.py — 扬声器播放 + +``` +gRPC AudioChunk → SpeakerDriver.play_chunk() → aplay stdin → ALSA 设备 +``` + +- 调用 `aplay -D hw:X,Y -f S16_LE -r 24000 -c 1 -t raw` +- 惰性启动：首次 `play_chunk()` 时才启动 aplay +- 自动重启：BrokenPipeError 时自动重启 aplay 并重试 +- 线程安全：内部使用 threading.Lock + +### node.py — 主入口 + +启动流程（参照 tiago_bridge 模式）： + +``` +1. scan_alsa_devices() → 发现硬件 +2. find_default_mic/speaker() → 选择设备 +3. MicDriver / SpeakerDriver() → 创建驱动实例 +4. auto-pick ports → 自动分配 gRPC 端口 +5. RegisterNode → Atlas 注册 +6. DeclareInterface × 2 → 声明 mic + speaker 接口 +7. daemon threads → 启动心跳 + 2 个 gRPC server +8. main thread sleep → 阻塞等待 +``` + +## 作为参考模板 + +新建 prm driver 时复制此包并修改： + +| 替换项 | 本包值 | 你的包 | +|--------|--------|--------| +| proto 消息类型 | AudioConfig, AudioChunk | 你的硬件数据类型 | +| gRPC 服务 | PrmAudioMic, PrmAudioSpeaker | 你的设备接口 | +| Contract ID | robonix/prm/audio/* | robonix/prm/你的设备/* | +| 扫描工具 | arecord -l / aplay -l | 你的设备发现方式 | +| 驱动类 | MicDriver, SpeakerDriver | 你的设备驱动 | +| 环境变量前缀 | AUDIO_ | 你的设备前缀 | +| manifest node ID | com.robonix.prm.audio | com.robonix.prm.你的设备 | + +保留不变的部分： +- `_ensure_proto_gen()` — proto stub 查找逻辑 +- `_register_with_atlas()` — Atlas 注册 + 心跳 +- daemon 线程模式 — 每个 interface 一个 gRPC server 线程 +- `robonix_manifest.yaml` 结构 +- `scripts/build.sh` proto 生成 + +## Atlas 集成 + +- **RegisterNode**: `com.robonix.prm.audio`，namespace `robonix/prm/audio`，kind `primitive` +- **DeclareInterface**: mic（server-stream）+ speaker（client-stream） +- **心跳**: 每 15 秒发送 NodeHeartbeat +- **降级**: Atlas 不可用时自动独立运行 diff --git a/rust/examples/packages/audio_driver/audio_driver/__init__.py b/rust/examples/packages/audio_driver/audio_driver/__init__.py new file mode 100644 index 000000000..7a53603b9 --- /dev/null +++ b/rust/examples/packages/audio_driver/audio_driver/__init__.py @@ -0,0 +1,15 @@ +# Robonix Audio Driver (robonix/prm/audio) +# +# Primitive-layer audio driver for microphone capture and speaker playback. +# Auto-discovers ALSA devices and provides gRPC streaming interfaces for +# both input (mic) and output (speaker). +# +# This package is the REFERENCE TEMPLATE for all future prm driver packages. +# +# Key modules: +# node.py — Main entry point (Atlas registration + gRPC servers) +# alsa_utils.py — ALSA device scanning + driver registry +# mic_driver.py — Microphone capture via arecord subprocess +# speaker_driver.py — Speaker playback via aplay subprocess +# +# Entry point: python -m audio_driver.node diff --git a/rust/examples/packages/audio_driver/audio_driver/alsa_utils.py b/rust/examples/packages/audio_driver/audio_driver/alsa_utils.py new file mode 100644 index 000000000..0889724f8 --- /dev/null +++ b/rust/examples/packages/audio_driver/audio_driver/alsa_utils.py @@ -0,0 +1,264 @@ +"""ALSA device auto-detection and driver registry for the audio_driver package. + +This module provides two main capabilities: + +1. **Device scanning**: Parses `arecord -l` and `aplay -l` output to discover + all ALSA audio devices on the system, including USB microphones and speakers. + +2. **Driver registry**: A pluggable system for hardware-specific drivers. + The DefaultAlsaDriver handles any standard ALSA device. To support + specialized hardware (e.g. ReSpeaker multi-channel mic arrays), create + a subclass of AudioDeviceDriver and call register_driver(). + +Architecture position: This is a utility module used by node.py during +startup to discover available audio hardware before creating MicDriver +and SpeakerDriver instances. + +Usage: + from audio_driver.alsa_utils import scan_alsa_devices, find_default_mic + + devices = scan_alsa_devices() + mic = find_default_mic(devices) + if mic: + print(f"Using mic: {mic.device_id} ({mic.name})") + + # Register a custom driver: + from audio_driver.alsa_utils import AudioDeviceDriver, register_driver + class MyDriver(AudioDeviceDriver): + def detect(self, devices): ... + def name(self): return "My Custom Driver" + register_driver(MyDriver()) +""" +import re +import subprocess +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + +log = logging.getLogger(__name__) + + +@dataclass +class AlsaDeviceInfo: + """Represents a single ALSA audio device discovered on the system. + + Attributes: + card: ALSA card index (0 = usually built-in, 1+ = often USB). + device: ALSA device index within the card. + name: Human-readable device name from arecord/aplay (e.g. "USB Audio Device"). + device_id: ALSA hardware device string (e.g. "hw:1,0") used by arecord/aplay -D. + is_input: True if this device can capture audio (found in arecord -l). + is_output: True if this device can play audio (found in aplay -l). + """ + card: int + device: int + name: str # e.g. "USB Audio Device" + device_id: str # e.g. "hw:1,0" + is_input: bool = False + is_output: bool = False + + +# ── ALSA device scanning ───────────────────────────────────────────────── + +def _parse_alsa_devices(output: str, is_input: bool) -> list[AlsaDeviceInfo]: + """Parse the output of `arecord -l` or `aplay -l` into AlsaDeviceInfo list. + + Matches lines like: + card 1: Device [USB Audio Device], device 0: USB Audio ... + + Args: + output: Combined stdout+stderr from the ALSA listing command. + is_input: True for arecord (input devices), False for aplay (output). + + Returns: + List of AlsaDeviceInfo, one per matched device line. + """ + devices = [] + # Match lines like: card 1: Device [USB Audio Device], device 0: ... + pattern = re.compile( + r"card\s+(\d+):\s+(.+?)\s*\[(.+?)\],\s+device\s+(\d+):" + ) + for match in pattern.finditer(output): + card = int(match.group(1)) + name = match.group(3).strip() + device = int(match.group(4)) + device_id = f"hw:{card},{device}" + devices.append(AlsaDeviceInfo( + card=card, device=device, name=name, + device_id=device_id, is_input=is_input, is_output=not is_input, + )) + return devices + + +def scan_alsa_devices() -> list[AlsaDeviceInfo]: + """Scan all ALSA input and output devices on the system. + + Runs `arecord -l` (input/mic devices) and `aplay -l` (output/speaker + devices) via subprocess, parses the output, and merges results by + (card, device) key. A device that appears in both lists will have + both is_input=True and is_output=True. + + Returns: + List of AlsaDeviceInfo. Empty if no ALSA devices found or if + arecord/aplay are not installed. + + Typical output on a machine with USB mic + built-in speaker: + [AlsaDeviceInfo(card=0, device=0, name="HDA Intel PCH", ...), + AlsaDeviceInfo(card=1, device=0, name="USB Audio Device", ...)] + """ + merged: dict[tuple[int, int], AlsaDeviceInfo] = {} + + for cmd, is_input in [("arecord", True), ("aplay", False)]: + try: + result = subprocess.run( + [cmd, "-l"], capture_output=True, text=True, timeout=5) + output = result.stdout + result.stderr + except FileNotFoundError: + log.warning("%s not found, skipping %s scan", + cmd, "input" if is_input else "output") + continue + except subprocess.TimeoutExpired: + log.warning("%s -l timed out", cmd) + continue + + for dev in _parse_alsa_devices(output, is_input): + key = (dev.card, dev.device) + if key in merged: + existing = merged[key] + if is_input: + existing.is_input = True + else: + existing.is_output = True + else: + merged[key] = dev + + devices = list(merged.values()) + for d in devices: + log.debug("ALSA device: %s (%s) in=%s out=%s", + d.device_id, d.name, d.is_input, d.is_output) + return devices + + +def find_default_mic(devices: list[AlsaDeviceInfo] | None = None) -> AlsaDeviceInfo | None: + """Find the best default microphone device. + + Selection priority: + 1. USB input devices (name contains "usb" or card >= 1) + 2. Any input device (first available) + + Args: + devices: Pre-scanned device list. If None, calls scan_alsa_devices(). + + Returns: + AlsaDeviceInfo for the best mic, or None if no input device exists. + """ + if devices is None: + devices = scan_alsa_devices() + inputs = [d for d in devices if d.is_input] + if not inputs: + return None + # Prefer USB devices (typically have "USB" in name or card >= 1) + usb = [d for d in inputs if "usb" in d.name.lower() or d.card >= 1] + return usb[0] if usb else inputs[0] + + +def find_default_speaker(devices: list[AlsaDeviceInfo] | None = None) -> AlsaDeviceInfo | None: + """Find the best default speaker device. + + Selection priority: + 1. USB output devices (name contains "usb" or card >= 1) + 2. Any output device (first available) + + Args: + devices: Pre-scanned device list. If None, calls scan_alsa_devices(). + + Returns: + AlsaDeviceInfo for the best speaker, or None if no output device exists. + """ + if devices is None: + devices = scan_alsa_devices() + outputs = [d for d in devices if d.is_output] + if not outputs: + return None + usb = [d for d in outputs if "usb" in d.name.lower() or d.card >= 1] + return usb[0] if usb else outputs[0] + + +# ── Driver Registry ───────────────────────────────────────────────────── +# Pluggable driver system for hardware-specific audio device detection. +# The DefaultAlsaDriver is always registered and accepts all ALSA devices. +# To add support for specialized hardware: +# 1. Subclass AudioDeviceDriver +# 2. Implement detect() to filter supported devices +# 3. Call register_driver(MyDriver()) at module load time +# The node.py startup will iterate through registered drivers to match hardware. + +class AudioDeviceDriver(ABC): + """Abstract base class for audio device drivers. + + A driver's job is to identify which ALSA devices it supports (via detect()) + and provide a human-readable name. Future extensions may add methods for + device-specific configuration (e.g. ReSpeaker channel mapping). + + Example — custom driver for ReSpeaker mic arrays: + class RespeakerDriver(AudioDeviceDriver): + def detect(self, devices): + return [d for d in devices if "ReSpeaker" in d.name] + def name(self): + return "ReSpeaker Driver" + register_driver(RespeakerDriver()) + """ + + @abstractmethod + def detect(self, devices: list[AlsaDeviceInfo]) -> list[AlsaDeviceInfo]: + """Return the subset of devices this driver supports. + + Args: + devices: All ALSA devices found on the system. + + Returns: + Filtered list of devices this driver can handle. + """ + ... + + @abstractmethod + def name(self) -> str: + """Human-readable driver name for logging.""" + ... + + +class DefaultAlsaDriver(AudioDeviceDriver): + """Default ALSA driver — accepts all standard ALSA devices. + + This is the catch-all driver registered by default. It doesn't filter + any devices, making all detected hardware available for use. + """ + + def detect(self, devices: list[AlsaDeviceInfo]) -> list[AlsaDeviceInfo]: + return devices # accepts everything + + def name(self) -> str: + return "ALSA Default" + + +# Global driver registry — DefaultAlsaDriver is always available. +# Additional drivers are appended via register_driver(). +_DRIVER_REGISTRY: list[AudioDeviceDriver] = [DefaultAlsaDriver()] + + +def register_driver(driver: AudioDeviceDriver) -> None: + """Register a hardware-specific audio driver. + + Registered drivers are available for device detection. Call this at + module import time (top-level) or in your package's __init__.py. + + Args: + driver: An AudioDeviceDriver subclass instance. + """ + _DRIVER_REGISTRY.append(driver) + log.info("Registered audio driver: %s", driver.name()) + + +def get_drivers() -> list[AudioDeviceDriver]: + """Return a copy of the current driver registry.""" + return list(_DRIVER_REGISTRY) diff --git a/rust/examples/packages/audio_driver/audio_driver/mic_driver.py b/rust/examples/packages/audio_driver/audio_driver/mic_driver.py new file mode 100644 index 000000000..4f6854982 --- /dev/null +++ b/rust/examples/packages/audio_driver/audio_driver/mic_driver.py @@ -0,0 +1,172 @@ +"""Microphone capture driver using arecord subprocess. + +This module provides MicDriver, which wraps the ALSA `arecord` command-line +tool to capture raw PCM audio from a microphone device. It produces audio +chunks (dicts) at a configurable rate for consumption by the gRPC servicer. + +Architecture position: Used by node.py to create a mic capture pipeline: + ALSA device → arecord subprocess → MicDriver.read_chunk() → gRPC AudioChunk + +Why subprocess (not PyAudio/sounddevice): + - arecord is universally available on Linux ALSA systems + - No Python C-extension compilation needed (no portaudio dependency) + - Simple and reliable for fixed-format capture + +Supported formats: + - 8-bit: S8 + - 16-bit: S16_LE (default, standard for ASR) + - 24-bit: S24_LE + - 32-bit: S32_LE + +Usage: + driver = MicDriver(device_id="hw:1,0", sample_rate=16000, channels=1) + driver.start() + while True: + chunk = driver.read_chunk() + if chunk is None: + break + # chunk = {"timestamp_ns": int, "data": bytes, "sequence": int, "duration_s": float} + process(chunk) + driver.stop() +""" +import subprocess +import time +import logging + +log = logging.getLogger(__name__) + +# ALSA format name mapping — bits_per_sample → arecord -f argument. +_ALSA_FORMATS = { + 16: "S16_LE", # 16-bit signed little-endian (standard for ASR) + 24: "S24_LE", # 24-bit signed little-endian + 32: "S32_LE", # 32-bit signed little-endian + 8: "S8", # 8-bit signed +} + + +class MicDriver: + """Captures audio from an ALSA device via arecord subprocess. + + Launches `arecord -D -f -r -c -t raw` + and reads fixed-size chunks from its stdout. Each chunk is wrapped in a + dict with metadata (timestamp, sequence number, duration). + + Thread safety: NOT thread-safe. Designed for single-reader use within + a dedicated gRPC servicer thread (one client per MicDriver instance). + + Lifecycle: + 1. __init__() — configure parameters, no subprocess yet + 2. start() — launch arecord subprocess + 3. read_chunk() — read one chunk (blocking, returns dict or None) + 4. stop() — terminate subprocess + + Args: + device_id: ALSA device string (e.g. "hw:1,0", "plughw:0,0"). + sample_rate: Capture sample rate in Hz (default 16000 for ASR). + channels: Number of channels (default 1 = mono). + bits_per_sample: Bits per sample, 8/16/24/32 (default 16). + chunk_duration_s: Duration of each read_chunk() call in seconds + (default 0.1 = 100ms). Determines chunk size in bytes: + bytes = sample_rate × channels × (bits/8) × duration_s + """ + + def __init__( + self, + device_id: str, + sample_rate: int = 16000, + channels: int = 1, + bits_per_sample: int = 16, + chunk_duration_s: float = 0.1, + ): + self.device_id = device_id + self.sample_rate = sample_rate + self.channels = channels + self.bits_per_sample = bits_per_sample + self.chunk_duration_s = chunk_duration_s + + # Pre-compute exact chunk size in bytes for read_chunk() + self._bytes_per_sample = bits_per_sample // 8 + self._chunk_bytes = ( + int(sample_rate * chunk_duration_s) + * channels * self._bytes_per_sample + ) + self._process: subprocess.Popen | None = None + self._sequence = 0 + + def start(self) -> None: + """Launch the arecord subprocess for audio capture. + + Builds and executes the arecord command with the configured parameters. + The subprocess stdout is read in fixed-size chunks by read_chunk(). + + Raises: + ValueError: If bits_per_sample is not in _ALSA_FORMATS. + FileNotFoundError: If arecord is not installed on the system. + """ + fmt = _ALSA_FORMATS.get(self.bits_per_sample) + if fmt is None: + raise ValueError(f"Unsupported bits_per_sample={self.bits_per_sample}") + + cmd = [ + "arecord", + "-D", self.device_id, + "-f", fmt, + "-r", str(self.sample_rate), + "-c", str(self.channels), + "-t", "raw", + ] + log.info("Starting mic capture: %s", " ".join(cmd)) + self._process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._sequence = 0 + + def read_chunk(self) -> dict | None: + """Read one audio chunk from arecord stdout (blocking). + + Reads exactly _chunk_bytes from the subprocess stdout. This blocks + until enough audio data is available (real-time capture rate). + + Returns: + Dict with keys: + timestamp_ns (int): Nanoseconds since epoch when chunk was read. + data (bytes): Raw PCM audio bytes (length = _chunk_bytes). + sequence (int): Monotonically increasing chunk counter. + duration_s (float): Configured chunk duration in seconds. + Returns None if the subprocess has ended or read returns less + data than expected (partial read / EOF). + """ + if self._process is None: + return None + + data = self._process.stdout.read(self._chunk_bytes) + if not data or len(data) < self._chunk_bytes: + return None + + chunk = { + "timestamp_ns": int(time.time() * 1e9), + "data": data, + "sequence": self._sequence, + "duration_s": self.chunk_duration_s, + } + self._sequence += 1 + return chunk + + def stop(self) -> None: + """Stop the arecord subprocess gracefully. + + Sends SIGTERM, waits up to 5 seconds, then SIGKILL if necessary. + Safe to call multiple times. + """ + if self._process: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + self._process = None + log.info("Mic capture stopped") + + @property + def is_running(self) -> bool: + """True if arecord subprocess is alive and capturing.""" + return self._process is not None and self._process.poll() is None diff --git a/rust/examples/packages/audio_driver/audio_driver/node.py b/rust/examples/packages/audio_driver/audio_driver/node.py new file mode 100644 index 000000000..96d2d61b7 --- /dev/null +++ b/rust/examples/packages/audio_driver/audio_driver/node.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Audio driver node — the prm-layer audio primitive for Robonix. + +Architecture position: robonix/prm/audio (primitive / hardware layer) + - Sits BELOW the srv layer (speech_service) and ABOVE the hardware + - Auto-discovers ALSA audio devices (microphones and speakers) + - Provides two gRPC interfaces for audio I/O: + 1. PrmAudioMic.Stream — server-streaming, outputs mic audio + 2. PrmAudioSpeaker.Stream — client-streaming, plays received audio + +This node follows the **tiago_bridge pattern** for prm driver packages: + 1. Auto-scan ALSA devices (arecord -l / aplay -l) + 2. RegisterNode with Atlas control plane + 3. DeclareInterface for mic + speaker (2 contracts) + 4. Start daemon threads: + - Heartbeat thread (15s interval to Atlas) + - Mic gRPC server thread (PrmAudioMicServicer) + - Speaker gRPC server thread (PrmAudioSpeakerServicer) + 5. Main thread blocks until KeyboardInterrupt + +This package is designed as a **REFERENCE TEMPLATE** for all future prm +driver packages. The same pattern should be replicated for camera, +lidar, IMU, and other hardware drivers. + +Environment variables (all optional — sensible defaults provided): + ROBONIX_ATLAS Atlas control-plane address (default: localhost:50051) + ROBONIX_NODE_ID Node ID for Atlas registration (default: com.robonix.prm.audio) + AUDIO_MIC_DEVICE Override mic ALSA device (default: auto-detect via arecord -l) + AUDIO_MIC_SAMPLE_RATE Mic capture sample rate in Hz (default: 16000) + AUDIO_MIC_CHANNELS Mic capture channels (default: 1 = mono) + AUDIO_MIC_BITS Mic bits per sample (default: 16) + AUDIO_MIC_CHUNK_MS Mic chunk duration in milliseconds (default: 100) + AUDIO_MIC_PORT Mic gRPC server port (default: 0 = auto-pick) + AUDIO_SPEAKER_DEVICE Override speaker ALSA device (default: auto-detect) + AUDIO_SPEAKER_SAMPLE_RATE Speaker playback rate in Hz (default: 24000) + AUDIO_SPEAKER_CHANNELS Speaker channels (default: 1) + AUDIO_SPEAKER_BITS Speaker bits per sample (default: 16) + AUDIO_SPEAKER_PORT Speaker gRPC server port (default: 0 = auto-pick) + AUDIO_DRIVER_STANDALONE Set to "1" or "true" to skip Atlas registration +""" +import json +import os +import sys +import time +import logging +import threading +from concurrent import futures +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="[audio-driver] %(levelname)s %(message)s") +log = logging.getLogger(__name__) + +# ── Proto stub resolution ────────────────────────────────────────────────── +# Walks up from this file's directory looking for proto_gen/ containing the +# generated audio_driver_pb2.py. Adds it to sys.path so the proto imports work +# regardless of the current working directory. + +def _ensure_proto_gen() -> None: + d = Path(__file__).resolve().parent + while d.parent != d: + pg = d / "proto_gen" + if pg.is_dir() and (pg / "audio_driver_pb2.py").exists(): + sys.path.insert(0, str(pg)) + return + d = d.parent + +_ensure_proto_gen() + +import grpc +import audio_driver_pb2 as pb +import audio_driver_pb2_grpc as pb_grpc + +from audio_driver.alsa_utils import scan_alsa_devices, find_default_mic, find_default_speaker +from audio_driver.mic_driver import MicDriver +from audio_driver.speaker_driver import SpeakerDriver + +# ── gRPC Servicers ──────────────────────────────────────────────────────── +# Each interface (mic, speaker) gets its own servicer class that wraps +# the corresponding driver. The servicers are registered on separate gRPC +# servers running in daemon threads. + +class PrmAudioMicServicer(pb_grpc.PrmAudioMicServicer): + """Implements contract robonix/prm/audio/mic — server-streaming mic audio. + + When a client calls Stream(): + 1. Starts the MicDriver (arecord subprocess) + 2. Loops reading chunks from the driver + 3. Wraps each chunk as an AudioChunk proto and yields it + 4. On client disconnect, stops the MicDriver + + Thread safety: One MicDriver per servicer instance. Multiple clients + connecting simultaneously will share the same mic stream (second client + gets the already-running stream). For exclusive access, the gRPC layer + should be configured with one stream at a time. + """ + + def __init__(self, mic_driver: MicDriver): + self.mic_driver = mic_driver + + def Stream(self, request, context): + log.info("Mic stream client connected") + self.mic_driver.start() + try: + while context.is_active(): + chunk = self.mic_driver.read_chunk() + if chunk is None: + break + yield pb.AudioChunk( + timestamp_ns=chunk["timestamp_ns"], + data=chunk["data"], + sequence=chunk["sequence"], + duration_s=chunk["duration_s"], + ) + finally: + self.mic_driver.stop() + log.info("Mic stream client disconnected") + + +class PrmAudioSpeakerServicer(pb_grpc.PrmAudioSpeakerServicer): + """Implements contract robonix/prm/audio/speaker — client-streaming playback. + + When a client calls Stream() with a stream of SpeakerChunkRequest: + 1. Reads audio_config from the first message (if present) and logs it + 2. For each chunk with data, calls speaker_driver.play_chunk() + 3. On stream end, returns SpeakerStreamResponse(ok=True) + + The SpeakerDriver handles lazy-starting aplay and auto-restart on errors. + """ + + def __init__(self, speaker_driver: SpeakerDriver): + self.speaker_driver = speaker_driver + + def Stream(self, request_iterator, context): + log.info("Speaker stream client connected") + config_set = False + try: + for req in request_iterator: + if req.audio_config and not config_set: + log.info("Speaker config: rate=%d ch=%d bits=%d enc=%s", + req.audio_config.sample_rate_hz, + req.audio_config.channels, + req.audio_config.bits_per_sample, + req.audio_config.encoding) + config_set = True + + if req.chunk and req.chunk.data: + self.speaker_driver.play_chunk(bytes(req.chunk.data)) + finally: + log.info("Speaker stream client disconnected") + + return pb.SpeakerStreamResponse(ok=True) + + +# ── Atlas registration (optional) ───────────────────────────────────────── +# Follows the tiago_bridge pattern exactly: +# 1. RegisterNode — register as a primitive under robonix/prm/audio +# 2. DeclareInterface × 2 — declare mic and speaker endpoints +# 3. Heartbeat thread — send NodeHeartbeat every 15 seconds + +def _register_with_atlas(mic_port: int, speaker_port: int) -> None: + """Register this node with the Atlas control plane. Non-fatal on failure. + + If robonix_runtime_pb2 is not importable (standalone deployment), or + if Atlas is unreachable, logs a warning and continues. The driver is + fully functional without Atlas — it just won't be discoverable. + + Registration details: + - Node ID: com.robonix.prm.audio (configurable via ROBONIX_NODE_ID) + - Namespace: robonix/prm/audio + - Kind: primitive (hardware driver) + - Interface 1: "mic" — contract robonix/prm/audio/mic, server-stream + - Interface 2: "speaker" — contract robonix/prm/audio/speaker, client-stream + + Args: + mic_port: gRPC port the mic server is listening on. + speaker_port: gRPC port the speaker server is listening on. + """ + try: + import robonix_runtime_pb2 as rpb + import robonix_runtime_pb2_grpc as rpb_grpc + except ImportError: + log.warning("robonix_runtime_pb2 not found, skipping Atlas registration") + return + + atlas_addr = os.environ.get("ROBONIX_ATLAS", "localhost:50051") + log.info("Connecting to Atlas at %s ...", atlas_addr) + + try: + channel = grpc.insecure_channel(atlas_addr) + stub = rpb_grpc.RobonixRuntimeStub(channel) + node_id = os.environ.get("ROBONIX_NODE_ID", "com.robonix.prm.audio") + + stub.RegisterNode(rpb.RegisterNodeRequest( + node_id=node_id, + namespace="robonix/prm/audio", + kind="primitive", + )) + log.info("Registered node %s with Atlas", node_id) + + # Mic interface + stub.DeclareInterface(rpb.DeclareInterfaceRequest( + node_id=node_id, name="mic", + supported_transports=["grpc"], + metadata_json=json.dumps({ + "transport": "grpc", + "contract": {"idl_type": "protobuf", "mode": "server_stream"}, + }), + listen_port=mic_port, + contract_id="robonix/prm/audio/mic", + )) + log.info("Declared robonix/prm/audio/mic on port %d", mic_port) + + # Speaker interface + stub.DeclareInterface(rpb.DeclareInterfaceRequest( + node_id=node_id, name="speaker", + supported_transports=["grpc"], + metadata_json=json.dumps({ + "transport": "grpc", + "contract": {"idl_type": "protobuf", "mode": "client_stream"}, + }), + listen_port=speaker_port, + contract_id="robonix/prm/audio/speaker", + )) + log.info("Declared robonix/prm/audio/speaker on port %d", speaker_port) + + # Heartbeat + def _heartbeat(): + while True: + time.sleep(15.0) + try: + stub.NodeHeartbeat(rpb.NodeHeartbeatRequest(node_id=node_id)) + except Exception as e: + log.warning("Heartbeat failed: %s", e) + + threading.Thread(target=_heartbeat, daemon=True).start() + channel.close() + + except grpc.RpcError as e: + log.warning("Atlas not available (%s), running standalone", e.code()) + + +# ── gRPC server helpers ─────────────────────────────────────────────────── +# Each interface runs in its own daemon thread with a separate gRPC server. +# This isolates mic and speaker so a problem with one doesn't affect the other. + +def _run_mic_server(port: int, mic_driver: MicDriver) -> None: + """Start and run the mic gRPC server (blocks until terminated). + + Creates a gRPC server with 4 worker threads, registers the + PrmAudioMicServicer, binds to 0.0.0.0:, and blocks. + + Args: + port: TCP port to listen on. + mic_driver: MicDriver instance to stream audio from. + """ + server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) + pb_grpc.add_PrmAudioMicServicer_to_server( + PrmAudioMicServicer(mic_driver), server) + server.add_insecure_port(f"0.0.0.0:{port}") + server.start() + log.info("Mic gRPC server on port %d", port) + server.wait_for_termination() + + +def _run_speaker_server(port: int, speaker_driver: SpeakerDriver) -> None: + """Start and run the speaker gRPC server (blocks until terminated). + + Creates a gRPC server with 4 worker threads, registers the + PrmAudioSpeakerServicer, binds to 0.0.0.0:, and blocks. + + Args: + port: TCP port to listen on. + speaker_driver: SpeakerDriver instance to play audio through. + """ + server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) + pb_grpc.add_PrmAudioSpeakerServicer_to_server( + PrmAudioSpeakerServicer(speaker_driver), server) + server.add_insecure_port(f"0.0.0.0:{port}") + server.start() + log.info("Speaker gRPC server on port %d", port) + server.wait_for_termination() + + +# ── Main ────────────────────────────────────────────────────────────────── +# Entry point. Called via `python -m audio_driver.node` or via the +# robonix_manifest.yaml start command. + +def main() -> None: + """Start the audio driver node. + + Startup sequence: + 1. Discover ALSA devices via scan_alsa_devices() + 2. Find mic and speaker devices (auto-detect or env override) + 3. Create MicDriver and SpeakerDriver instances + 4. Auto-pick free ports for mic and speaker gRPC servers + 5. Optionally register with Atlas (RegisterNode + DeclareInterface) + 6. Start mic gRPC server in a daemon thread + 7. Start speaker gRPC server in a daemon thread + 8. Block main thread until KeyboardInterrupt + """ + log.info("Starting audio driver") + + # 1. Discover ALSA devices + devices = scan_alsa_devices() + for d in devices: + log.info("Found ALSA device: %s (%s) in=%s out=%s", + d.device_id, d.name, d.is_input, d.is_output) + + mic_device = None + mic_device_str = os.environ.get("AUDIO_MIC_DEVICE", "") + if mic_device_str: + mic_device = type("Dev", (), {"device_id": mic_device_str})() + log.info("Mic device overridden: %s", mic_device_str) + else: + mic_info = find_default_mic(devices) + if mic_info: + mic_device = mic_info + log.info("Auto-detected mic: %s (%s)", mic_info.device_id, mic_info.name) + else: + log.warning("No microphone found") + + speaker_device = None + speaker_device_str = os.environ.get("AUDIO_SPEAKER_DEVICE", "") + if speaker_device_str: + speaker_device = type("Dev", (), {"device_id": speaker_device_str})() + log.info("Speaker device overridden: %s", speaker_device_str) + else: + speaker_info = find_default_speaker(devices) + if speaker_info: + speaker_device = speaker_info + log.info("Auto-detected speaker: %s (%s)", speaker_info.device_id, speaker_info.name) + else: + log.warning("No speaker found") + + # 2. Create drivers + mic_driver = None + if mic_device: + mic_driver = MicDriver( + device_id=mic_device.device_id, + sample_rate=int(os.environ.get("AUDIO_MIC_SAMPLE_RATE", "16000")), + channels=int(os.environ.get("AUDIO_MIC_CHANNELS", "1")), + bits_per_sample=int(os.environ.get("AUDIO_MIC_BITS", "16")), + chunk_duration_s=int(os.environ.get("AUDIO_MIC_CHUNK_MS", "100")) / 1000.0, + ) + + speaker_driver = None + if speaker_device: + speaker_driver = SpeakerDriver( + device_id=speaker_device.device_id, + sample_rate=int(os.environ.get("AUDIO_SPEAKER_SAMPLE_RATE", "24000")), + channels=int(os.environ.get("AUDIO_SPEAKER_CHANNELS", "1")), + bits_per_sample=int(os.environ.get("AUDIO_SPEAKER_BITS", "16")), + ) + + # 3. Pick ports + mic_port = int(os.environ.get("AUDIO_MIC_PORT", "0")) + speaker_port = int(os.environ.get("AUDIO_SPEAKER_PORT", "0")) + + if mic_port == 0: + # Auto-pick: bind temporary server to get a free port + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("0.0.0.0", 0)) + mic_port = s.getsockname()[1] + s.close() + + if speaker_port == 0: + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("0.0.0.0", 0)) + speaker_port = s.getsockname()[1] + s.close() + + # 4. Atlas registration (optional) + standalone = os.environ.get("AUDIO_DRIVER_STANDALONE", "").strip() in ("1", "true") + if not standalone: + _register_with_atlas(mic_port, speaker_port) + + # 5. Start gRPC servers + if mic_driver: + threading.Thread( + target=_run_mic_server, args=(mic_port, mic_driver), + daemon=True, name="mic-grpc", + ).start() + + if speaker_driver: + threading.Thread( + target=_run_speaker_server, args=(speaker_port, speaker_driver), + daemon=True, name="speaker-grpc", + ).start() + + log.info("Audio driver ready (mic=%s, speaker=%s)", + f"port {mic_port}" if mic_driver else "none", + f"port {speaker_port}" if speaker_driver else "none") + + # 6. Block main thread + try: + while True: + time.sleep(1.0) + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/rust/examples/packages/audio_driver/audio_driver/speaker_driver.py b/rust/examples/packages/audio_driver/audio_driver/speaker_driver.py new file mode 100644 index 000000000..a3b874834 --- /dev/null +++ b/rust/examples/packages/audio_driver/audio_driver/speaker_driver.py @@ -0,0 +1,154 @@ +"""Speaker playback driver using aplay subprocess. + +This module provides SpeakerDriver, which wraps the ALSA `aplay` command-line +tool to play raw PCM audio through a speaker device. It writes audio data to +aplay's stdin and handles automatic restart on pipe errors. + +Architecture position: Used by node.py to create a speaker playback pipeline: + gRPC AudioChunk → SpeakerDriver.play_chunk() → aplay stdin → ALSA speaker + +Key design decisions: + - Lazy subprocess start: aplay is started on first play_chunk() call + - Thread-safe: uses threading.Lock for concurrent play_chunk() calls + - Auto-restart: on BrokenPipeError (aplay crashed/ended), re-launches + automatically and retries the write + - No buffering: each play_chunk() call flushes immediately to aplay stdin + +Usage: + driver = SpeakerDriver(device_id="hw:1,0", sample_rate=24000) + driver.play_chunk(pcm_audio_bytes) + driver.play_chunk(more_audio_bytes) + driver.stop() +""" +import subprocess +import threading +import logging + +log = logging.getLogger(__name__) + +# ALSA format name mapping — bits_per_sample → aplay -f argument. +_ALSA_FORMATS = { + 16: "S16_LE", # 16-bit signed little-endian (standard for TTS output) + 24: "S24_LE", # 24-bit signed little-endian + 32: "S32_LE", # 32-bit signed little-endian + 8: "S8", # 8-bit signed +} + + +class SpeakerDriver: + """Plays raw PCM audio through an ALSA device via aplay subprocess. + + Launches `aplay -D -f -r -c -t raw` + on first play_chunk() call. Audio data is written to aplay's stdin. + If the aplay process dies (BrokenPipeError), it is automatically + restarted and the audio data is re-sent. + + Thread safety: YES — all writes are protected by a threading.Lock. + Safe for concurrent use from multiple gRPC handler threads. + + Lifecycle: + 1. __init__() — configure parameters, no subprocess yet + 2. play_chunk(data) — lazy-start aplay, write audio data + 3. play_chunk(data) — write more audio (reuses same process) + 4. stop() — close stdin and terminate aplay + + Args: + device_id: ALSA device string (e.g. "hw:1,0", "plughw:0,0"). + sample_rate: Playback sample rate in Hz (default 24000 for TTS). + channels: Number of channels (default 1 = mono). + bits_per_sample: Bits per sample, 8/16/24/32 (default 16). + """ + + def __init__( + self, + device_id: str, + sample_rate: int = 24000, + channels: int = 1, + bits_per_sample: int = 16, + ): + self.device_id = device_id + self.sample_rate = sample_rate + self.channels = channels + self.bits_per_sample = bits_per_sample + self._process: subprocess.Popen | None = None + self._lock = threading.Lock() + + def _ensure_aplay(self) -> None: + """Lazy-start the aplay subprocess if not already running. + + Called internally by play_chunk() before each write. If aplay has + exited (poll() returns non-None), a new process is started. + + Raises: + ValueError: If bits_per_sample is not supported. + """ + if self._process and self._process.poll() is None: + return + + fmt = _ALSA_FORMATS.get(self.bits_per_sample) + if fmt is None: + raise ValueError(f"Unsupported bits_per_sample={self.bits_per_sample}") + + cmd = [ + "aplay", + "-D", self.device_id, + "-f", fmt, + "-r", str(self.sample_rate), + "-c", str(self.channels), + "-t", "raw", + ] + log.info("Starting speaker playback: %s", " ".join(cmd)) + self._process = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + + def play_chunk(self, audio_data: bytes) -> None: + """Write audio data to aplay stdin for immediate playback. + + Thread-safe: acquires a lock so concurrent calls are serialized. + Auto-restart: if writing fails (BrokenPipeError), terminates the + old aplay process, starts a new one, and retries the write. + + Args: + audio_data: Raw PCM audio bytes. Must match the configured + format (sample_rate, channels, bits_per_sample). Any + length is acceptable — aplay buffers internally. + """ + with self._lock: + self._ensure_aplay() + try: + self._process.stdin.write(audio_data) + self._process.stdin.flush() + except BrokenPipeError: + log.warning("aplay pipe broken, restarting") + self._process.terminate() + self._process.wait(timeout=3) + self._process = None + self._ensure_aplay() + self._process.stdin.write(audio_data) + self._process.stdin.flush() + + def stop(self) -> None: + """Stop the aplay subprocess gracefully. + + Closes stdin (signals aplay to finish), then sends SIGTERM. + Waits up to 5 seconds, then SIGKILL if necessary. + Safe to call multiple times. + """ + with self._lock: + if self._process: + try: + self._process.stdin.close() + except Exception: + pass + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + self._process = None + log.info("Speaker playback stopped") + + @property + def is_running(self) -> bool: + """True if aplay subprocess is alive and ready for audio.""" + return self._process is not None and self._process.poll() is None diff --git a/rust/examples/packages/audio_driver/proto/audio_driver.proto b/rust/examples/packages/audio_driver/proto/audio_driver.proto new file mode 100644 index 000000000..f13d8f47f --- /dev/null +++ b/rust/examples/packages/audio_driver/proto/audio_driver.proto @@ -0,0 +1,93 @@ +// ────────────────────────────────────────────────────────────────────────── +// Audio Driver — gRPC service definitions for mic capture & speaker playback +// +// Architecture position: robonix/prm/audio (primitive / hardware layer) +// - Auto-scans ALSA devices (arecord -l / aplay -l) +// - Captures raw PCM from microphone → streams AudioChunk upward +// - Receives AudioChunk from upper-layer services → plays via speaker +// - Registers with Atlas as primitive node under robonix/prm/audio +// +// This package serves as a REFERENCE TEMPLATE for all future prm driver +// packages. Key patterns to replicate: +// 1. Self-contained proto (inlined types, no cross-package imports) +// 2. ALSA auto-detection via subprocess (arecord/aplay) +// 3. Driver registry for hardware extensibility (AudioDeviceDriver ABC) +// 4. Atlas registration + heartbeat (tiago_bridge pattern) +// 5. Daemon threads for gRPC servers (one per interface) +// +// Build: scripts/build.sh generates *_pb2.py / *_pb2_grpc.py into proto_gen/ +// ────────────────────────────────────────────────────────────────────────── + +syntax = "proto3"; +package robonix.audio_driver; + +// ── Shared types ───────────────────────────────────────────────────────── +// Inlined for standalone build. Decoupled from speech_service.proto types. + +// Describes the audio format used by a stream. +message AudioConfig { + string encoding = 1; // PCM encoding name, e.g. "pcm_s16le", "pcm_s24le" + uint32 sample_rate_hz = 2; // sample rate in Hz (mic default: 16000, speaker default: 24000) + uint32 channels = 3; // 1 = mono, 2 = stereo + uint32 bits_per_sample = 4; // bits per sample (8, 16, 24, 32) +} + +// A single time-stamped audio chunk. Produced by mic capture, consumed by +// speaker playback. Sequence numbers are monotonic per stream for ordering. +message AudioChunk { + uint64 timestamp_ns = 1; // capture/playback timestamp in nanoseconds since epoch + bytes data = 2; // raw PCM bytes (no headers, no container) + uint32 sequence = 3; // monotonically increasing chunk counter + float duration_s = 4; // duration of this chunk in seconds (e.g. 0.1) +} + +// ── Mic interface (topic_out: driver produces, consumers subscribe) ────── +// +// Data flow: ALSA mic → arecord subprocess → MicDriver → gRPC stream → caller +// +// The mic driver captures at a fixed format (default 16kHz mono s16le). +// If audio_config is provided, it is logged but currently the driver captures +// at its configured rate (see environment overrides in node.py). + +message MicStreamRequest { + AudioConfig audio_config = 1; // optional: desired capture format. If empty, uses defaults. +} + +// ── Speaker interface (topic_in: consumers push, driver plays) ────────── +// +// Data flow: caller → gRPC client stream → SpeakerDriver → aplay subprocess → ALSA speaker +// +// The first message SHOULD include audio_config to set the playback format. +// Subsequent messages carry chunk.data for continuous playback. + +message SpeakerChunkRequest { + AudioChunk chunk = 1; // audio data to play + AudioConfig audio_config = 2; // optional: sent once in first message to configure playback +} + +message SpeakerStreamResponse { + bool ok = 1; // true if all chunks played successfully + string error = 2; // non-empty on failure +} + +// ── Services ──────────────────────────────────────────────────────────── +// Two separate services for mic and speaker, each registered with Atlas +// as a distinct interface under the robonix/prm/audio namespace. + +// Microphone capture service. +// Contract ID: robonix/prm/audio/mic +// Transport: gRPC server-streaming +// Mode: caller sends one request, server streams AudioChunk continuously +// until the caller disconnects or the mic stops. +service PrmAudioMic { + rpc Stream(MicStreamRequest) returns (stream AudioChunk); +} + +// Speaker playback service. +// Contract ID: robonix/prm/audio/speaker +// Transport: gRPC client-streaming +// Mode: caller streams audio chunks, driver plays them in real time. +// Returns a single response when the caller closes the stream. +service PrmAudioSpeaker { + rpc Stream(stream SpeakerChunkRequest) returns (SpeakerStreamResponse); +} diff --git a/rust/examples/packages/audio_driver/requirements.txt b/rust/examples/packages/audio_driver/requirements.txt new file mode 100644 index 000000000..b28c1f6a7 --- /dev/null +++ b/rust/examples/packages/audio_driver/requirements.txt @@ -0,0 +1,3 @@ +grpcio>=1.60 +grpcio-tools>=1.60 +protobuf>=4.24 diff --git a/rust/examples/packages/audio_driver/robonix_manifest.yaml b/rust/examples/packages/audio_driver/robonix_manifest.yaml new file mode 100644 index 000000000..2c146d153 --- /dev/null +++ b/rust/examples/packages/audio_driver/robonix_manifest.yaml @@ -0,0 +1,19 @@ +manifestVersion: 1 + +build: + script: scripts/build.sh + +package: + id: com.robonix.example.audio_driver + name: audio_driver + version: 0.1.0 + vendor: robonix + description: Audio driver (mic capture + speaker playback) for robonix/prm/audio contracts + license: MulanPSL-2.0 + +nodes: + - id: com.robonix.prm.audio + type: python + start: | + export PYTHONPATH="$(pwd)/proto_gen:${PYTHONPATH:-}" + exec python -m audio_driver.node diff --git a/rust/examples/packages/audio_driver/scripts/build.sh b/rust/examples/packages/audio_driver/scripts/build.sh new file mode 100644 index 000000000..a98ca6f85 --- /dev/null +++ b/rust/examples/packages/audio_driver/scripts/build.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MulanPSL-2.0 +set -euo pipefail +PKG="${RBNX_PACKAGE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +PROTO_SRC="$PKG/proto" +OUT_DIR="$PKG/proto_gen" + +mkdir -p "$OUT_DIR" + +python -m grpc_tools.protoc \ + -I"$PROTO_SRC" \ + --python_out="$OUT_DIR" \ + --grpc_python_out="$OUT_DIR" \ + "$PROTO_SRC/audio_driver.proto" + +echo "[build] done." diff --git a/rust/examples/packages/speech_service/.gitignore b/rust/examples/packages/speech_service/.gitignore new file mode 100644 index 000000000..a10042696 --- /dev/null +++ b/rust/examples/packages/speech_service/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +proto_gen/ +rbnx-build/ +*.egg-info/ diff --git a/rust/examples/packages/speech_service/README.md b/rust/examples/packages/speech_service/README.md new file mode 100644 index 000000000..4bab7470e --- /dev/null +++ b/rust/examples/packages/speech_service/README.md @@ -0,0 +1,137 @@ +# Speech Service + +语音交互服务 — Robonix srv 层，提供 ASR（语音识别）、TTS（语音合成）、Dialog（语音对话）三类 gRPC 接口。 + +## 架构定位 + +``` +┌─────────────────────────────────────────────┐ +│ Application Layer │ +├─────────────────────────────────────────────┤ +│ speech_service ← 你在这里 (robonix/srv/speech) │ +├─────────────────────────────────────────────┤ +│ audio_driver (robonix/prm/audio) │ +├─────────────────────────────────────────────┤ +│ Hardware (ALSA / USB Mic/Speaker) │ +└─────────────────────────────────────────────┘ +``` + +- **上层**：接收应用层文本请求（TTS）或返回识别结果（ASR） +- **下层**：从 audio_driver 接收原始 PCM 音频流 +- **自适应性**：自动将任意采样率/声道/编码转为 16kHz mono pcm_s16le，调用方无需预处理 + +## gRPC 接口 + +| 服务 | RPC | 模式 | 后端 | 说明 | +|------|-----|------|------|------| +| SpeechAsr | Recognize | Unary | Whisper (GPU FP16) | 一次性识别，适合完整语句 | +| SpeechAsr | RecognizeStream | Bidi-stream | FunASR Paraformer | 流式识别，600ms 粒度 | +| SpeechTts | Synthesize | Unary | Edge TTS | 一次性合成，返回完整 MP3 | +| SpeechTts | SynthesizeStream | Server-stream | Edge TTS | 流式合成，边生成边返回 | +| SpeechDialog | StartDialog | Server-stream | — | 语音对话会话管理 | + +## 目录结构 + +``` +speech_service/ +├── proto/ +│ └── speech_service.proto # 自包含 gRPC 定义（内联 AudioConfig/AudioChunk） +├── proto_gen/ # build 时生成的 *_pb2.py（git 忽略） +├── scripts/ +│ └── build.sh # proto 代码生成 +├── speech_service/ +│ ├── __init__.py # 包入口 +│ ├── service.py # gRPC servicer + 后端引擎 + main() +│ └── audio_utils.py # 音频格式自适应（任意格式 → 16kHz mono s16le） +├── robonix_manifest.yaml # Robonix 包描述 +├── requirements.txt +└── .gitignore +``` + +## 快速启动 + +### 1. 安装依赖 + +```bash +pip install -r requirements.txt +# 核心依赖：grpcio, grpcio-tools, transformers, torch, funasr, edge-tts, scipy, numpy +``` + +### 2. 生成 proto stubs + +```bash +bash scripts/build.sh +``` + +### 3. 启动服务 + +```bash +# 正常模式（需要 GPU + 模型权重） +export PYTHONPATH=$(pwd)/proto_gen:${PYTHONPATH:-} +python -m speech_service.service + +# CI 模式（不需要 GPU/模型，返回 mock 结果） +SPEECH_CI_MODE=1 python -m speech_service.service + +# 指定端口 +SPEECH_PORT=50060 python -m speech_service.service + +# 跳过 Atlas 注册 +SPEECH_STANDALONE=1 python -m speech_service.service +``` + +### 4. 通过 Robonix 启动 + +```bash +rbnx run com.robonix.example.speech_service +``` + +## 环境变量 + +| 变量 | 默认值 | 说明 | +|------|--------|------| +| `ASR_MODEL` | `whisper-large-merged` | Whisper 模型路径 | +| `ASR_DEVICE` | `cuda` | Torch 设备（cuda/cpu） | +| `ASR_CHUNK_LENGTH` | `30.0` | Whisper 长音频分片长度（秒） | +| `ASR_BATCH_SIZE` | `4` | Whisper 批处理大小 | +| `FUNASR_MODEL` | `paraformer-zh-streaming` | FunASR 流式模型 | +| `FUNASR_CHUNK_SIZE` | `[0,10,5]` | Paraformer chunk_size 参数 | +| `TTS_VOICE` | `zh-CN-XiaoxiaoNeural` | Edge TTS 声音名称 | +| `ROBONIX_ATLAS` | `localhost:50051` | Atlas 控制面地址 | +| `SPEECH_PORT` | `0`（自动分配） | gRPC 监听端口 | +| `SPEECH_BIND_ADDR` | `0.0.0.0` | gRPC 绑定地址 | +| `SPEECH_CI_MODE` | — | 设为 `1` 启用 mock 模式 | + +## 音频自适应 + +`audio_utils.py` 处理管线： + +``` +输入音频（任意格式） + │ + ├─ 1. 解码 → float32 numpy [-1, 1] + │ 支持: pcm_s16le, pcm_f32le, pcm_s32le, pcm_u8, pcm_s24le, wav + │ + ├─ 2. 多声道混合 → 单声道 + │ reshape(-1, channels).mean(axis=1) + │ + ├─ 3. 重采样 → 16kHz + │ scipy.signal.resample_poly（多相 FIR 滤波器） + │ + └─ 4. 编码 → pcm_s16le +``` + +调用方无需关心音频格式，服务端自动处理 8kHz/16kHz/48kHz/stereo 等输入。 + +## Atlas 集成 + +启动时可选注册到 Atlas 控制面： + +- **RegisterNode**: `com.robonix.services.speech`，namespace `robonix/srv/speech`，kind `service` +- **DeclareInterface × 5**: asr, asr_stream, tts, tts_stream, dialog + +Atlas 不可用时自动降级为独立运行模式。 + +## Proto 设计 + +`speech_service.proto` 是**自包含**的 — AudioConfig 和 AudioChunk 类型内联定义，不依赖外部 proto 文件。与 `audio_driver.proto` 中的同名类型解耦，各自独立演进。 diff --git a/rust/examples/packages/speech_service/proto/speech_service.proto b/rust/examples/packages/speech_service/proto/speech_service.proto new file mode 100644 index 000000000..f546d9fa3 --- /dev/null +++ b/rust/examples/packages/speech_service/proto/speech_service.proto @@ -0,0 +1,154 @@ +// ────────────────────────────────────────────────────────────────────────── +// Speech Service — gRPC service definitions for ASR / TTS / Dialog +// +// Architecture position: robonix/srv/speech (service layer) +// - Receives audio from lower-layer prm drivers (e.g. audio_driver) +// - Auto-adapts any input sample_rate / channels / encoding to 16kHz mono s16le +// - Outputs synthesized audio (MP3) for downstream playback +// +// Self-contained: all message types are inlined (no external proto imports). +// Build: scripts/build.sh generates *_pb2.py / *_pb2_grpc.py into proto_gen/ +// ────────────────────────────────────────────────────────────────────────── + +syntax = "proto3"; +package robonix.speech_service; + +// ── Shared types ───────────────────────────────────────────────────────── +// Inlined here so the package builds without cross-package proto dependencies. +// These mirror the types in audio_driver.proto but are intentionally decoupled. + +// Describes an audio stream's format. Used by both ASR (to describe input) +// and TTS (to describe output). +message AudioConfig { + string encoding = 1; // e.g. "pcm_s16le", "pcm_f32le", "mp3", "wav" + uint32 sample_rate_hz = 2; // e.g. 16000, 24000, 44100, 48000 + uint32 channels = 3; // 1 = mono, 2 = stereo + uint32 bits_per_sample = 4; // 8, 16, 24, or 32 +} + +// A single time-stamped audio chunk, used in streaming RPCs. +message AudioChunk { + uint64 timestamp_ns = 1; // nanoseconds since epoch (clock_gettime) + bytes data = 2; // raw PCM bytes (length = samples × channels × bytes_per_sample) + uint32 sequence = 3; // monotonically increasing counter per stream + float duration_s = 4; // chunk duration in seconds (e.g. 0.1 for 100ms) +} + +// ── ASR (Automatic Speech Recognition) ─────────────────────────────────── + +// One-shot recognition: send complete audio, receive transcription. +// The service adapts any sample_rate / channels / encoding to 16kHz mono s16le +// internally, so the caller does not need to pre-process audio. +message RecognizeRequest { + bytes audio_data = 1; // complete audio (PCM or WAV) + string encoding = 2; // audio encoding, default "pcm_s16le" + uint32 sample_rate_hz = 3; // input sample rate, default 16000 + string language = 4; // BCP-47 tag, e.g. "zh", "en", "ja" + uint32 channels = 5; // number of channels, default 1 (proto3 field addition, backward compatible) + uint32 bits_per_sample = 6; // bits per sample, default 16 (proto3 field addition, backward compatible) +} + +message RecognizeResponse { + string text = 1; // transcribed text (UTF-8) + float confidence = 2; // 0.0 – 1.0, estimated confidence + string error = 3; // non-empty on failure +} + +// Streaming recognition: client sends audio chunk-by-chunk, server yields +// partial and final transcriptions in real time. +// +// Protocol: +// 1. First message SHOULD set audio_config + language +// 2. Subsequent messages carry chunk.data +// 3. Client closes stream to signal end-of-utterance +// 4. Server sends final result with is_final=true +message RecognizeStreamRequest { + AudioConfig audio_config = 1; // sent in first message (or when config changes) + string language = 2; // BCP-47 tag + AudioChunk chunk = 3; // audio chunk data +} + +message RecognizeStreamResponse { + uint32 event_type = 1; // 0 = PARTIAL (intermediate), 1 = FINAL, 2 = ERROR + string text = 2; // transcribed text so far + float confidence = 3; // 0.0 – 1.0 + string language = 4; // detected or requested language + bool is_final = 5; // true on last response before stream closes + string error = 6; // non-empty when event_type = 2 +} + +// ASR service: two recognition modes. +service SpeechAsr { + // Unary: send complete audio buffer, get one transcription. + // Backend: Whisper (transformers pipeline, GPU FP16). + rpc Recognize (RecognizeRequest) returns (RecognizeResponse); + + // Bidirectional streaming: chunk-by-chunk audio → partial/final results. + // Backend: FunASR Paraformer-zh-streaming (600ms granularity, 300ms lookahead). + rpc RecognizeStream (stream RecognizeStreamRequest) returns (stream RecognizeStreamResponse); +} + +// ── TTS (Text-to-Speech) ──────────────────────────────────────────────── + +// One-shot synthesis: send text, receive complete audio. +message SynthesizeRequest { + string text = 1; // text to synthesize (UTF-8) + string language = 2; // BCP-47 tag for pronunciation hints + string voice = 3; // voice name, e.g. "zh-CN-XiaoxiaoNeural" + float speed = 4; // speech speed multiplier (1.0 = normal) +} + +message SynthesizeResponse { + bytes audio_data = 1; // synthesized audio (MP3 format) + string encoding = 2; // always "mp3" for Edge TTS + uint32 sample_rate_hz = 3; // always 24000 for Edge TTS + string error = 4; // non-empty on failure +} + +// Streaming synthesis: send text, receive audio in chunks. +// Useful for long text — playback can begin before synthesis completes. +message SynthesizeStreamRequest { + string text = 1; + string language = 2; + string voice = 3; + float speed = 4; +} + +message SynthesizeStreamResponse { + bytes audio_data = 1; // one MP3 audio chunk + string encoding = 2; // "mp3" + uint32 sample_rate_hz = 3; // 24000 + bool is_final = 4; // true on last chunk (audio_data empty) +} + +// TTS service: two synthesis modes. +service SpeechTts { + // Unary: send text, get complete MP3 audio. + // Backend: Edge TTS (Microsoft Cognitive Services, free tier). + rpc Synthesize (SynthesizeRequest) returns (SynthesizeResponse); + + // Server streaming: send text, receive MP3 chunks as they're generated. + rpc SynthesizeStream (SynthesizeStreamRequest) returns (stream SynthesizeStreamResponse); +} + +// ── Dialog (voice interaction session) ─────────────────────────────────── + +// Start a managed voice dialog session. The server maintains session state +// (IDLE → LISTENING → PROCESSING → SPEAKING → IDLE) and streams state +// updates back to the client. Intended for future VAD + turn-taking logic. +message StartDialogRequest { + string language = 1; // BCP-47 tag, default "zh-CN" + bool enable_vad = 2; // enable voice activity detection (reserved for future use) +} + +message StartDialogResponse { + string session_id = 1; // unique session identifier (UUID prefix) + uint32 state = 2; // session state: 0=IDLE, 1=LISTENING, 2=PROCESSING, 3=SPEAKING, 4=ERROR + string error = 3; // non-empty on error +} + +service SpeechDialog { + // Server streaming: opens a dialog session, streams state transitions. + // Session is automatically cleaned up when the client disconnects. + rpc StartDialog (StartDialogRequest) returns (stream StartDialogResponse); +} diff --git a/rust/examples/packages/speech_service/rbnx_speech_smoke.py b/rust/examples/packages/speech_service/rbnx_speech_smoke.py new file mode 100644 index 000000000..a57e38551 --- /dev/null +++ b/rust/examples/packages/speech_service/rbnx_speech_smoke.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""End-to-end integration test: Atlas → speech_service → audio_driver. + +Prerequisites: + - Atlas running on localhost:50051 + - speech_service proto stubs generated (proto_gen/) + - robonix_runtime proto stubs generated (proto_gen/) + +Run: + export PYTHONPATH=$(pwd)/proto_gen:$(pwd):${PYTHONPATH:-} + SPEECH_CI_MODE=1 python test_integration.py +""" +import os +import sys +import time +import socket +import logging +import threading + +logging.basicConfig(level=logging.INFO, format="[test] %(levelname)s %(message)s") +log = logging.getLogger(__name__) + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "proto_gen")) + +import grpc +import robonix_runtime_pb2 as rpb +import robonix_runtime_pb2_grpc as rpb_grpc +import speech_service_pb2 as spb +import speech_service_pb2_grpc as spb_grpc + +ATLAS_ADDR = os.environ.get("ROBONIX_ATLAS", "localhost:50051") + +# ── Helpers ────────────────────────────────────────────────────────────── + +def check_port(host, port, timeout=3): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((host, port)) + s.close() + return True + except Exception: + return False + + +def atlas_stub(): + channel = grpc.insecure_channel(ATLAS_ADDR) + return rpb_grpc.RobonixRuntimeStub(channel), channel + + +# ── Test 1: Atlas is reachable ────────────────────────────────────────── + +def test_atlas_alive(): + log.info("Test 1: Atlas alive check") + stub, ch = atlas_stub() + try: + # QueryNodes with empty filter should work even with nothing registered + resp = stub.QueryNodes(rpb.QueryNodesRequest()) + log.info(" Atlas responded: %d nodes registered", len(resp.nodes)) + ch.close() + return True + except grpc.RpcError as e: + log.error(" Atlas NOT reachable: %s", e) + ch.close() + return False + + +# ── Test 2: Register speech_service with Atlas ───────────────────────── + +def test_register_speech(): + log.info("Test 2: Register speech_service with Atlas") + stub, ch = atlas_stub() + + node_id = "com.robonix.services.speech" + contracts = [ + ("robonix/srv/speech/asr", "asr", "rpc"), + ("robonix/srv/speech/asr_stream", "asr_stream", "rpc_bidirectional_stream"), + ("robonix/srv/speech/tts", "tts", "rpc"), + ("robonix/srv/speech/tts_stream", "tts_stream", "rpc_server_stream"), + ("robonix/srv/speech/dialog", "dialog", "rpc_server_stream"), + ] + + try: + stub.RegisterNode(rpb.RegisterNodeRequest( + node_id=node_id, + namespace="robonix/srv/speech", + kind="service", + )) + log.info(" RegisterNode OK: %s", node_id) + + for cid, name, mode in contracts: + import json + meta = json.dumps({"transport": "grpc", "contract": {"idl_type": "protobuf", "mode": mode}}) + resp = stub.DeclareInterface(rpb.DeclareInterfaceRequest( + node_id=node_id, name=name, + supported_transports=["grpc"], + metadata_json=meta, listen_port=50060, + contract_id=cid, + )) + log.info(" DeclareInterface OK: %s → endpoint=%s", cid, resp.allocated_endpoint) + + ch.close() + return True + except grpc.RpcError as e: + log.error(" Registration FAILED: %s (%s)", e.code(), e.details()) + ch.close() + return False + + +# ── Test 3: Register audio_driver with Atlas ─────────────────────────── + +def test_register_audio(): + log.info("Test 3: Register audio_driver with Atlas") + stub, ch = atlas_stub() + + node_id = "com.robonix.prm.audio" + + try: + stub.RegisterNode(rpb.RegisterNodeRequest( + node_id=node_id, + namespace="robonix/prm/audio", + kind="primitive", + )) + log.info(" RegisterNode OK: %s", node_id) + + for cid, name, mode, port in [ + ("robonix/prm/audio/mic", "mic", "server_stream", 50070), + ("robonix/prm/audio/speaker", "speaker", "client_stream", 50071), + ]: + import json + meta = json.dumps({"transport": "grpc", "contract": {"idl_type": "protobuf", "mode": mode}}) + resp = stub.DeclareInterface(rpb.DeclareInterfaceRequest( + node_id=node_id, name=name, + supported_transports=["grpc"], + metadata_json=meta, listen_port=port, + contract_id=cid, + )) + log.info(" DeclareInterface OK: %s → endpoint=%s", cid, resp.allocated_endpoint) + + ch.close() + return True + except grpc.RpcError as e: + log.error(" Registration FAILED: %s (%s)", e.code(), e.details()) + ch.close() + return False + + +# ── Test 4: Discover speech_service via QueryNodes ───────────────────── + +def test_discover_speech(): + log.info("Test 4: Discover speech_service via QueryNodes") + stub, ch = atlas_stub() + + results = {} + for cid in [ + "robonix/srv/speech/asr", + "robonix/srv/speech/tts", + "robonix/srv/speech/dialog", + "robonix/prm/audio/mic", + "robonix/prm/audio/speaker", + ]: + try: + resp = stub.QueryNodes(rpb.QueryNodesRequest(contract_id=cid)) + if resp.nodes: + n = resp.nodes[0] + ifaces = [i.contract_id for i in n.interfaces] + log.info(" FOUND: %s → node=%s interfaces=%s", cid, n.node_id, ifaces) + results[cid] = True + else: + log.error(" NOT FOUND: %s", cid) + results[cid] = False + except grpc.RpcError as e: + log.error(" QueryNodes FAILED for %s: %s", cid, e) + results[cid] = False + + ch.close() + return all(results.values()) + + +# ── Test 5: Start speech_service and call RPCs ───────────────────────── + +def test_speech_service_rpc(): + log.info("Test 5: Start speech_service (CI mode) and call RPCs") + + # Start speech_service in CI mode in background + env = os.environ.copy() + env["SPEECH_CI_MODE"] = "1" + env["SPEECH_PORT"] = "50060" + env["PYTHONPATH"] = f"{os.path.dirname(__file__)}:{os.path.dirname(__file__)}/proto_gen:{env.get('PYTHONPATH', '')}" + + import subprocess + proc = subprocess.Popen( + [sys.executable, "-m", "speech_service.service"], + env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + log.info(" speech_service started (PID=%d)", proc.pid) + + # Wait for it to be ready + for _ in range(20): + if check_port("127.0.0.1", 50060): + break + time.sleep(0.5) + else: + log.error(" speech_service did not start on port 50060") + proc.terminate() + return False + + log.info(" speech_service listening on port 50060") + + # Call Recognize + channel = grpc.insecure_channel("localhost:50060") + asr_stub = spb_grpc.SpeechAsrStub(channel) + + resp = asr_stub.Recognize(spb.RecognizeRequest( + audio_data=b"\x00" * 3200, + encoding="pcm_s16le", + sample_rate_hz=16000, + language="zh", + )) + log.info(" Recognize: text=%r confidence=%.1f error=%r", resp.text, resp.confidence, resp.error) + asr_ok = resp.error == "" and len(resp.text) > 0 + + # Call Synthesize + tts_stub = spb_grpc.SpeechTtsStub(channel) + resp = tts_stub.Synthesize(spb.SynthesizeRequest( + text="你好世界", + language="zh", + )) + log.info(" Synthesize: audio=%d bytes encoding=%s error=%r", len(resp.audio_data), resp.encoding, resp.error) + tts_ok = resp.error == "" and len(resp.audio_data) > 0 + + channel.close() + proc.terminate() + proc.wait(timeout=5) + return asr_ok and tts_ok + + +# ── Main ──────────────────────────────────────────────────────────────── + +def main(): + log.info("=" * 60) + log.info("Robonix speech integration test suite") + log.info("=" * 60) + + results = {} + + results["1_atlas_alive"] = test_atlas_alive() + results["2_register_speech"] = test_register_speech() + results["3_register_audio"] = test_register_audio() + results["4_discover"] = test_discover_speech() + results["5_rpc"] = test_speech_service_rpc() + + log.info("=" * 60) + log.info("RESULTS:") + all_pass = True + for name, ok in results.items(): + status = "PASS" if ok else "FAIL" + log.info(" %s: %s", name, status) + if not ok: + all_pass = False + log.info("=" * 60) + + if all_pass: + log.info("ALL TESTS PASSED") + else: + log.error("SOME TESTS FAILED") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/rust/examples/packages/speech_service/requirements.txt b/rust/examples/packages/speech_service/requirements.txt new file mode 100644 index 000000000..3fe233128 --- /dev/null +++ b/rust/examples/packages/speech_service/requirements.txt @@ -0,0 +1,11 @@ +# Speech Service dependencies +torch>=2.0 +transformers>=4.30 +funasr>=1.0 +edge-tts>=6.1 +opencc-python-reimplemented>=0.1.7 +grpcio>=1.60 +grpcio-tools>=1.60 +protobuf>=4.24 +scipy>=1.10 +numpy>=1.24 diff --git a/rust/examples/packages/speech_service/robonix_manifest.yaml b/rust/examples/packages/speech_service/robonix_manifest.yaml new file mode 100644 index 000000000..900ff45d8 --- /dev/null +++ b/rust/examples/packages/speech_service/robonix_manifest.yaml @@ -0,0 +1,23 @@ +# Robonix Speech Service — ASR (Whisper) + TTS (Edge TTS) + Dialog Manager. +# Data plane: gRPC servicers for speech/asr, speech/tts, speech/asr_stream, +# speech/tts_stream, speech/dialog contracts. +# Host build: scripts/build.sh runs rbnx codegen for proto stubs. +manifestVersion: 1 + +build: + script: scripts/build.sh + +package: + id: com.robonix.example.speech_service + name: speech_service + version: 0.1.0 + vendor: robonix + description: Voice interaction service (ASR + TTS + Dialog) registered under robonix/srv/speech + license: MulanPSL-2.0 + +nodes: + - id: com.robonix.services.speech + type: python + start: | + export PYTHONPATH="$(pwd)/proto_gen:${PYTHONPATH:-}" + exec python -m speech_service.service diff --git a/rust/examples/packages/speech_service/scripts/build.sh b/rust/examples/packages/speech_service/scripts/build.sh new file mode 100755 index 000000000..bb743a4bb --- /dev/null +++ b/rust/examples/packages/speech_service/scripts/build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MulanPSL-2.0 +set -euo pipefail +PKG="${RBNX_PACKAGE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +PROTO_SRC="$PKG/proto" +PROTO_DEPS="$PKG/../../../crates/robonix-interfaces/robonix_proto" +OUT_DIR="$PKG/proto_gen" + +FLAGS=() +[[ "${RBNX_BUILD_CLEAN:-}" == "1" ]] && FLAGS+=(--clean) + +# Step 1: rbnx codegen (generates IDL-based protos: asr.proto, tts.proto, etc.) +rbnx codegen -p "$PKG" "${FLAGS[@]}" || true + +# Step 2: Generate gRPC service stubs from speech_service.proto +mkdir -p "$OUT_DIR" +python -m grpc_tools.protoc \ + -I"$OUT_DIR" \ + -I"$PROTO_DEPS" \ + -I"$PROTO_SRC" \ + --python_out="$OUT_DIR" \ + --grpc_python_out="$OUT_DIR" \ + "$PROTO_SRC/speech_service.proto" + +echo "[build] done." diff --git a/rust/examples/packages/speech_service/speech_service/__init__.py b/rust/examples/packages/speech_service/speech_service/__init__.py new file mode 100644 index 000000000..9d7edf588 --- /dev/null +++ b/rust/examples/packages/speech_service/speech_service/__init__.py @@ -0,0 +1,11 @@ +# Robonix Speech Service (robonix/srv/speech) +# +# Provides ASR (speech-to-text), TTS (text-to-speech), and Dialog (voice +# session) services over gRPC. Part of the srv (service) layer in the +# Robonix architecture, sitting above the prm (primitive) audio driver. +# +# Key modules: +# service.py — gRPC servicers and main entry point +# audio_utils — audio format adaptation (any → 16kHz mono s16le) +# +# Entry point: python -m speech_service.service diff --git a/rust/examples/packages/speech_service/speech_service/audio_utils.py b/rust/examples/packages/speech_service/speech_service/audio_utils.py new file mode 100644 index 000000000..f5803885b --- /dev/null +++ b/rust/examples/packages/speech_service/speech_service/audio_utils.py @@ -0,0 +1,221 @@ +"""Audio format adaptation utilities for the speech service. + +This module is the upper-layer adaptation layer between the audio_driver +(prm, hardware) and the ASR backends (Whisper, FunASR). Its job is to +normalize ANY audio format the lower layer sends into the exact format +the ASR backends expect: **16 kHz mono pcm_s16le**. + +Pipeline (adapt_audio): + 1. Decode raw bytes → float32 numpy array in [-1, 1] + 2. Mix multi-channel → mono (mean of channels) + 3. Resample to 16 kHz using scipy.signal.resample_poly (polyphase FIR) + 4. Re-encode to pcm_s16le (int16, 2 bytes per sample) + +Supported input encodings: + - pcm_s16le — 16-bit signed little-endian (most common ALSA output) + - pcm_f32le — 32-bit float little-endian + - pcm_s32le — 32-bit signed little-endian + - pcm_u8 — 8-bit unsigned + - pcm_s24le — 24-bit signed little-endian (3 bytes per sample) + - wav — RIFF/WAV container (any PCM sub-format) + +Usage: + from speech_service.audio_utils import adapt_audio, parse_audio_config + + # From a gRPC RecognizeRequest: + audio, enc = adapt_audio( + request.audio_data, + encoding=request.encoding or "pcm_s16le", + sample_rate=request.sample_rate_hz or 16000, + channels=request.channels or 1, + bits_per_sample=request.bits_per_sample or 16, + ) + # audio is now 16kHz mono pcm_s16le bytes, enc is "pcm_s16le" + + # From a streaming RecognizeStreamRequest with AudioConfig: + config = parse_audio_config(req.audio_config) + adapted, _ = adapt_audio(chunk_data, **config) +""" +import io +import math +import wave +import logging + +log = logging.getLogger(__name__) + +# Target format — all ASR backends (Whisper, FunASR) expect this exact format. +# The adapt_audio() function converts any input to these parameters. +TARGET_SAMPLE_RATE = 16000 # Hz — standard telephony/ASR sample rate +TARGET_CHANNELS = 1 # mono + + +def parse_audio_config(audio_config) -> dict: + """Extract audio format fields from a proto AudioConfig message. + + Accepts any object (proto message, dict, or None) and returns a dict + with all four format parameters, using sensible defaults for missing + or zero-valued fields (proto3 defaults all fields to zero/empty). + + Args: + audio_config: Proto AudioConfig object or None. Attributes read: + encoding (str), sample_rate_hz (int), channels (int), + bits_per_sample (int). + + Returns: + dict with keys: encoding, sample_rate, channels, bits_per_sample. + Default values: "pcm_s16le", 16000, 1, 16. + """ + return { + "encoding": getattr(audio_config, "encoding", None) or "pcm_s16le", + "sample_rate": getattr(audio_config, "sample_rate_hz", None) or 16000, + "channels": getattr(audio_config, "channels", None) or 1, + "bits_per_sample": getattr(audio_config, "bits_per_sample", None) or 16, + } + + +def adapt_audio( + audio_bytes: bytes, + encoding: str = "pcm_s16le", + sample_rate: int = 16000, + channels: int = 1, + bits_per_sample: int = 16, +) -> tuple[bytes, str]: + """Normalise audio to 16 kHz mono pcm_s16le for ASR backends. + + This is the core adaptation function. It converts any input format + to the exact format required by Whisper and FunASR backends. + + Pipeline: + 1. _decode_to_float() — decode raw bytes → float32 ndarray in [-1, 1] + 2. Mono mix — if channels > 1, reshape(-1, ch).mean(axis=1) + 3. _resample() — scipy.signal.resample_poly if sample_rate != 16000 + 4. Encode to pcm_s16le — clip + cast to int16 + tobytes() + + Args: + audio_bytes: Raw audio bytes (PCM or WAV container). + encoding: Audio encoding string. Supported: pcm_s16le, pcm_f32le, + pcm_s32le, pcm_u8, pcm_s24le, wav. Unknown encodings fall back + to pcm_s16le decoding with a warning. + sample_rate: Input sample rate in Hz. If different from 16000, + resampling is performed using polyphase FIR filters. + channels: Number of audio channels. If > 1, channels are mixed + to mono by averaging. + bits_per_sample: Bits per sample (8, 16, 24, or 32). Used by + pcm_s24le decoder; other encoders derive this from the format. + + Returns: + Tuple of (audio_bytes, encoding_string): + - audio_bytes: pcm_s16le bytes at 16 kHz mono + - encoding_string: always "pcm_s16le" + + Example: + # Convert 48kHz stereo audio to 16kHz mono + data, enc = adapt_audio(raw_bytes, "pcm_s16le", 48000, 2, 16) + # data is now 16kHz mono s16le, enc is "pcm_s16le" + """ + import numpy as np + + if len(audio_bytes) == 0: + return audio_bytes, encoding + + # 1. Decode to float32 numpy array + audio = _decode_to_float(audio_bytes, encoding, bits_per_sample) + + # 2. Multi-channel → mono + if channels > 1: + audio = audio.reshape(-1, channels).mean(axis=1) + + # 3. Resample if needed + if sample_rate != TARGET_SAMPLE_RATE: + audio = _resample(audio, sample_rate, TARGET_SAMPLE_RATE) + log.debug("Resampled %d Hz → %d Hz (%d samples)", + sample_rate, TARGET_SAMPLE_RATE, len(audio)) + + # 4. Encode back to pcm_s16le + pcm = (audio * 32768.0).clip(-32768, 32767).astype(np.int16).tobytes() + return pcm, "pcm_s16le" + + +def _decode_to_float(data: bytes, encoding: str, bits_per_sample: int) -> "np.ndarray": + """Decode raw audio bytes to a float32 numpy array normalized to [-1, 1]. + + Each encoding type is handled individually for correct normalization: + - pcm_s16le: int16 / 32768.0 + - pcm_f32le: already float32, assumed in [-1, 1] + - pcm_s32le: int32 / 2147483648.0 + - pcm_u8: uint8 / 128.0 - 1.0 (shift unsigned to signed range) + - pcm_s24le: manual 3-byte LE unpack + sign extension / 8388608.0 + + For unknown encodings, falls back to WAV parsing, then to pcm_s16le. + + Args: + data: Raw audio bytes. + encoding: Encoding string (e.g. "pcm_s16le"). + bits_per_sample: Bits per sample (used for pcm_s24le). + + Returns: + numpy float32 array with values in [-1, 1]. + """ + import numpy as np + + if encoding == "pcm_s16le": + return np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0 + + if encoding == "pcm_f32le": + return np.frombuffer(data, dtype=np.float32) + + if encoding == "pcm_s32le": + return np.frombuffer(data, dtype=np.int32).astype(np.float32) / 2147483648.0 + + if encoding == "pcm_u8": + return np.frombuffer(data, dtype=np.uint8).astype(np.float32) / 128.0 - 1.0 + + if encoding == "pcm_s24le": + # 24-bit LE: read 3 bytes per sample, manual unpack + n_samples = len(data) // 3 + raw = np.frombuffer(data, dtype=np.uint8).reshape(-1, 3) + # little-endian: LSB first + samples = (raw[:, 0].astype(np.int32) + | (raw[:, 1].astype(np.int32) << 8) + | (raw[:, 2].astype(np.int32) << 16)) + # Sign-extend from 24-bit + samples = np.where(samples >= 0x800000, samples - 0x1000000, samples) + return samples.astype(np.float32) / 8388608.0 + + # Fallback: try WAV decode + buf = io.BytesIO(data) + try: + with wave.open(buf, "rb") as wf: + frames = wf.readframes(wf.getnframes()) + return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + except wave.Error: + # Last resort: assume pcm_s16le + log.warning("Unknown encoding %s, treating as pcm_s16le", encoding) + return np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0 + + +def _resample(audio: "np.ndarray", orig_sr: int, target_sr: int) -> "np.ndarray": + """Resample audio using polyphase FIR filter (scipy.signal.resample_poly). + + This method is preferred over FFT-based resampling for audio because: + - It preserves the waveform shape (linear-phase FIR) + - It handles rational rate ratios exactly (no spectral leakage) + - It's fast for typical audio lengths + + The up/down factors are computed from the GCD of the two rates to + avoid unnecessarily large filter lengths. + + Args: + audio: float32 numpy array (mono, any length). + orig_sr: Original sample rate in Hz. + target_sr: Desired sample rate in Hz. + + Returns: + Resampled float32 numpy array at target_sr. + """ + from scipy.signal import resample_poly + + gcd = math.gcd(target_sr, orig_sr) + up = target_sr // gcd + down = orig_sr // gcd + return resample_poly(audio, up, down) diff --git a/rust/examples/packages/speech_service/speech_service/service.py b/rust/examples/packages/speech_service/speech_service/service.py new file mode 100644 index 000000000..d07115c94 --- /dev/null +++ b/rust/examples/packages/speech_service/speech_service/service.py @@ -0,0 +1,841 @@ +#!/usr/bin/env python3 +"""Speech service — the srv-layer voice interaction service for Robonix. + +Architecture position: robonix/srv/speech + - Sits ABOVE the prm layer (audio_driver) and BELOW the application layer + - Receives raw audio from audio_driver via gRPC, returns transcriptions + - Receives text from applications, returns synthesized audio (MP3) + +Provides 3 gRPC services (5 RPCs total): + + SpeechAsr (Automatic Speech Recognition): + - Recognize(req) → resp — one-shot, full-utterance transcription + - RecognizeStream(stream) → stream — real-time chunk-by-chunk transcription + + SpeechTts (Text-to-Speech): + - Synthesize(req) → resp — one-shot, returns complete MP3 audio + - SynthesizeStream(req) → stream — streaming, yields MP3 chunks as generated + + SpeechDialog (Voice Dialog Session): + - StartDialog(req) → stream — managed session with state transitions + +Backend engines: + - Whisper (transformers pipeline) — one-shot ASR, GPU FP16, high accuracy + - FunASR Paraformer-zh-streaming — streaming ASR, 600ms granularity + - Edge TTS (Microsoft, free) — TTS synthesis, zh-CN-XiaoxiaoNeural + +Audio adaptation: + The service auto-adapts any input format (sample_rate, channels, encoding) + to 16kHz mono pcm_s16le using speech_service.audio_utils.adapt_audio(). + Callers do NOT need to pre-process audio — the service handles it. + +Atlas integration: + On startup, the service optionally registers with the Atlas control plane + (RegisterNode + DeclareInterface × 5). If Atlas is unavailable, the service + runs in standalone mode. Set SPEECH_STANDALONE=1 to skip registration. + +Environment variables: + ASR_MODEL Whisper model path (default: whisper-large-merged) + ASR_DEVICE Torch device: cuda | cpu (default: cuda) + ASR_CHUNK_LENGTH Whisper chunk length in seconds (default: 30.0) + ASR_BATCH_SIZE Whisper batch size (default: 4) + FUNASR_MODEL FunASR model name or path (default: paraformer-zh-streaming) + FUNASR_CHUNK_SIZE Paraformer chunk_size as JSON (default: [0,10,5]) + TTS_VOICE Edge TTS voice name (default: zh-CN-XiaoxiaoNeural) + ROBONIX_ATLAS Atlas control-plane address (default: localhost:50051) + SPEECH_PORT gRPC listen port, 0 = auto-pick (default: 0) + SPEECH_BIND_ADDR gRPC bind address (default: 0.0.0.0) + SPEECH_CI_MODE Set to 1 for mock backends (no GPU/model needed) +""" +import io +import json +import os +import sys +import time +import uuid +import asyncio +import logging +import wave +from concurrent import futures +from pathlib import Path +from typing import Optional + +logging.basicConfig(level=logging.INFO, format="[speech-service] %(levelname)s %(message)s") +log = logging.getLogger(__name__) + +# ── Proto stub resolution ────────────────────────────────────────────────── +# Walks up from this file's directory looking for proto_gen/ containing the +# generated *_pb2.py files. Adds it to sys.path so "import speech_service_pb2" +# works regardless of the working directory. + +def _ensure_proto_gen() -> None: + d = Path(__file__).resolve().parent + while d.parent != d: + pg = d / "proto_gen" + if pg.is_dir() and (pg / "speech_service_pb2.py").exists(): + sys.path.insert(0, str(pg)) + return + d = d.parent + +_ensure_proto_gen() + +import grpc +import speech_service_pb2 as spb +import speech_service_pb2_grpc as spb_grpc + +# ── CI mock mode ────────────────────────────────────────────────────────── + +CI_MODE = os.environ.get("SPEECH_CI_MODE", "").strip() in ("1", "true", "yes") + +# ── ASR Backend (Whisper) ───────────────────────────────────────────────── + +class WhisperASRBackend: + """One-shot ASR backend using HuggingFace Whisper pipeline. + + Loads a Whisper model (default: whisper-large-merged) onto GPU with FP16. + Best suited for complete utterances — not real-time streaming. + + Pipeline: + raw PCM bytes → numpy float32 → transformers pipeline → text + + automatic traditional→simplified Chinese conversion (OpenCC t2s) + + Config (environment): + ASR_MODEL — local model path or HuggingFace model ID + ASR_DEVICE — "cuda" or "cpu" + ASR_CHUNK_LENGTH — long-form audio chunk length in seconds + ASR_BATCH_SIZE — batch size for pipeline inference + """ + + def __init__(self): + import torch + from transformers import pipeline as hf_pipeline + + model_path = os.environ.get( + "ASR_MODEL", + "/home/zzb/zzb_HDD/Project/LLM_for_ASR/model/whisper-large-merged", + ) + device = os.environ.get("ASR_DEVICE", "cuda") + chunk_length = float(os.environ.get("ASR_CHUNK_LENGTH", "30.0")) + batch_size = int(os.environ.get("ASR_BATCH_SIZE", "4")) + + log.info("Loading Whisper model from %s on %s ...", model_path, device) + self.pipe = hf_pipeline( + "automatic-speech-recognition", + model=model_path, + device=device, + torch_dtype=torch.float16, + ) + self.chunk_length_s = chunk_length + self.batch_size = batch_size + log.info("Whisper model loaded (chunk_length=%.1fs, batch_size=%d)", chunk_length, batch_size) + + def recognize(self, audio_bytes: bytes, encoding: str, sample_rate: int, language: str) -> dict: + """Run one-shot ASR on complete audio. + + Args: + audio_bytes: Raw PCM audio (already adapted to 16kHz mono pcm_s16le + by the servicer, but can handle other encodings too). + encoding: Audio encoding string (e.g. "pcm_s16le"). + sample_rate: Sample rate of the audio data. + language: BCP-47 language tag (reserved, Whisper auto-detects). + + Returns: + dict with "text" (str) and "confidence" (float, fixed 0.9). + """ + import numpy as np + + wav_data = self._to_wav_numpy(audio_bytes, encoding, sample_rate) + gen_kwargs = {"chunk_length_s": self.chunk_length_s, "batch_size": self.batch_size} + + result = self.pipe(wav_data, **gen_kwargs) + text = result.get("text", "") + + # Whisper outputs traditional Chinese by default; convert to simplified + text = self._t2s(text) + return {"text": text, "confidence": 0.9} + + @staticmethod + def _to_wav_numpy(audio_bytes: bytes, encoding: str, sample_rate: int): + """Convert raw audio bytes to numpy float32 array for the pipeline. + + Handles pcm_s16le, pcm_f32le, and falls back to WAV parsing via + the stdlib wave module for container formats. + """ + import numpy as np + + if encoding == "pcm_s16le": + audio = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0 + elif encoding == "pcm_f32le": + audio = np.frombuffer(audio_bytes, dtype=np.float32) + else: + buf = io.BytesIO(audio_bytes) + with wave.open(buf, "rb") as wf: + frames = wf.readframes(wf.getnframes()) + audio = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + return audio + + @staticmethod + def _t2s(text: str) -> str: + """Traditional Chinese -> Simplified Chinese.""" + try: + from opencc import OpenCC + cc = OpenCC("t2s") + return cc.convert(text) + except Exception: + return text + + +class MockASRBackend: + """CI mock ASR — returns a fixed canned response, no model loaded. + + Activated when SPEECH_CI_MODE=1. Useful for testing the gRPC layer + without requiring GPU or model weights. + """ + + def recognize(self, audio_bytes: bytes, encoding: str, sample_rate: int, language: str) -> dict: + return {"text": "[ci-mock] 你好世界", "confidence": 1.0} + + +# ── ASR Backend (FunASR Paraformer streaming) ───────────────────────────── + +class FunASRStreamingBackend: + """Streaming ASR backend using FunASR Paraformer-zh-streaming. + + Designed for real-time chunk-by-chunk recognition. The model processes + fixed-size audio chunks and maintains streaming state via a cache dict. + + Model parameters: + - Model: paraformer-zh-streaming (or FUNASR_MODEL env override) + - chunk_size: [0, 10, 5] — 0 left context, 10 current (600ms), 5 right (300ms) + - chunk_stride: 10 × 960 = 9600 samples = 600ms at 16kHz + - encoder_chunk_look_back: 4 (retain 4 past encoder chunks) + - decoder_chunk_look_back: 1 (retain 1 past decoder chunk) + + The cache dict is passed across recognize_chunk() calls within a single + stream session. Each RecognizeStream gRPC call gets its own cache. + + Config (environment): + FUNASR_MODEL — model name or local path + FUNASR_CHUNK_SIZE — JSON array [left, current, right] + """ + + def __init__(self): + from funasr import AutoModel + + model_name = os.environ.get("FUNASR_MODEL", "paraformer-zh-streaming") + chunk_size_str = os.environ.get("FUNASR_CHUNK_SIZE", "[0,10,5]") + try: + self.chunk_size = json.loads(chunk_size_str) + except json.JSONDecodeError: + self.chunk_size = [0, 10, 5] + + self.chunk_stride = self.chunk_size[1] * 960 + + log.info("Loading FunASR model %s (chunk_size=%s, stride=%d samples)...", + model_name, self.chunk_size, self.chunk_stride) + self.model = AutoModel(model=model_name) + log.info("FunASR model loaded.") + + def recognize_chunk( + self, + audio_chunk: bytes, + cache: dict, + is_final: bool = False, + encoding: str = "pcm_s16le", + sample_rate: int = 16000, + ) -> list[dict]: + """Process one audio chunk through the streaming Paraformer. + + Args: + audio_chunk: Raw PCM bytes (already adapted to 16kHz mono pcm_s16le). + Chunk size should be ~600ms (9600 samples × 2 bytes = 19200 bytes) + for optimal granularity, but any size works. + cache: Mutable dict maintained across chunks in one session. + The model stores encoder/decoder state here. Must be the same + dict for all chunks in one RecognizeStream call. + is_final: True on the last call to flush remaining state. + Send empty audio_chunk with is_final=True to get the final result. + encoding: Audio encoding (default pcm_s16le). + sample_rate: Sample rate (default 16000). + + Returns: + List of dicts, each with "text" (str) and "confidence" (float). + May be empty if the chunk is too short for a recognition event. + On is_final=True, returns the final transcription. + """ + import numpy as np + + if encoding == "pcm_s16le": + audio = np.frombuffer(audio_chunk, dtype=np.int16) + elif encoding == "pcm_f32le": + audio = (np.frombuffer(audio_chunk, dtype=np.float32) * 32768).astype(np.int16) + else: + buf = io.BytesIO(audio_chunk) + with wave.open(buf, "rb") as wf: + frames = wf.readframes(wf.getnframes()) + audio = np.frombuffer(frames, dtype=np.int16) + + results = self.model.generate( + input=audio, + cache=cache, + is_final=is_final, + chunk_size=self.chunk_size, + encoder_chunk_look_back=4, + decoder_chunk_look_back=1, + ) + + outputs = [] + for res in results: + if isinstance(res, dict): + text = res.get("text", "") + else: + text = str(res) + outputs.append({"text": text, "confidence": 0.9}) + return outputs + + +class MockASRStreamingBackend: + """CI mock streaming ASR — returns empty results during streaming, + canned result on is_final. No model loaded. + """ + + def recognize_chunk(self, audio_chunk, cache, is_final=False, encoding="pcm_s16le", sample_rate=16000): + if is_final: + return [{"text": "[ci-mock-stream] 你好世界", "confidence": 1.0}] + return [{"text": "", "confidence": 0.0}] + + +# ── TTS Backend (Edge TTS) ─────────────────────────────────────────────── + +class EdgeTTSBackend: + """TTS backend using Microsoft Edge TTS (free, no API key required). + + Uses the edge-tts Python package which wraps Microsoft's Cognitive + Services TTS endpoint. Outputs MP3 audio at 24kHz. + + Config (environment): + TTS_VOICE — voice name (default: zh-CN-XiaoxiaoNeural) + Other voices: zh-CN-YunxiNeural, en-US-JennyNeural, ja-JP-NanamiNeural + + Speed control: + speed parameter is a multiplier. Internally converted to Edge TTS + rate string format: "+N%" or "-N%" where N = |speed-1| × 100. + """ + + def __init__(self): + self.voice = os.environ.get("TTS_VOICE", "zh-CN-XiaoxiaoNeural") + log.info("Edge TTS initialized with voice=%s", self.voice) + + async def synthesize(self, text: str, voice: str = "", speed: float = 1.0) -> bytes: + """Synthesize text to complete MP3 audio bytes (one-shot). + + Args: + text: Text to synthesize. + voice: Voice name override (uses TTS_VOICE env if empty). + speed: Speed multiplier (1.0 = normal, 2.0 = 2x fast). + + Returns: + Complete MP3 audio bytes. + """ + import edge_tts + + v = voice or self.voice + sign = "+" if speed > 1 else "-" + rate = f"{sign}{int(abs(speed - 1) * 100)}%" if speed != 1.0 else "+0%" + communicate = edge_tts.Communicate(text, v, rate=rate) + chunks = [] + async for chunk in communicate.stream(): + if chunk["type"] == "audio": + chunks.append(chunk["data"]) + return b"".join(chunks) + + async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0): + """Async generator that yields MP3 audio chunks as they're generated. + + Useful for long text — the caller can start playback before the + full audio is ready. + + Args: + text: Text to synthesize. + voice: Voice name override. + speed: Speed multiplier. + + Yields: + bytes — individual MP3 audio chunks from Edge TTS streaming. + """ + import edge_tts + + v = voice or self.voice + sign = "+" if speed > 1 else "-" + rate = f"{sign}{int(abs(speed - 1) * 100)}%" if speed != 1.0 else "+0%" + communicate = edge_tts.Communicate(text, v, rate=rate) + async for chunk in communicate.stream(): + if chunk["type"] == "audio": + yield chunk["data"] + + +class MockTTSBackend: + """CI mock TTS — returns a minimal valid WAV file (silence). + Activated when SPEECH_CI_MODE=1. + """ + + async def synthesize(self, text: str, voice: str = "", speed: float = 1.0) -> bytes: + return b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x44\xac\x00\x00\x88X\x01\x00\x02\x00\x10\x00data\x00\x00\x00\x00" + + async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0): + data = await self.synthesize(text, voice, speed) + yield data + + +# ── Dialog Session Manager ──────────────────────────────────────────────── +# Manages the lifecycle of voice dialog sessions. Each StartDialog call +# creates a session that tracks state (IDLE → LISTENING → PROCESSING → +# SPEAKING) until the client disconnects. + +class DialogSession: + """Tracks state for a single voice dialog session. + + State machine: + IDLE → LISTENING → PROCESSING → SPEAKING → IDLE (loop) + Any state → ERROR (on failure) + + Attributes: + session_id: 8-char UUID prefix, unique per session. + language: BCP-47 language tag for this session. + enable_vad: Whether voice activity detection is active. + state: Current state string. + created_at: Unix timestamp of session creation. + """ + + def __init__(self, session_id: str, language: str, enable_vad: bool): + self.session_id = session_id + self.language = language + self.enable_vad = enable_vad + self.state = "IDLE" # IDLE | LISTENING | PROCESSING | SPEAKING | ERROR + self.created_at = time.time() + + STATE_MAP = {"IDLE": 0, "LISTENING": 1, "PROCESSING": 2, "SPEAKING": 3, "ERROR": 4} + + +class DialogManager: + """Manages multiple concurrent voice dialog sessions. + + Thread-safe via dict operations (single-threaded gRPC server pool). + Sessions are created by StartDialog and removed when the client + disconnects (servicer finally block). + """ + + def __init__(self): + self.sessions: dict[str, DialogSession] = {} + + def create_session(self, language: str, enable_vad: bool) -> DialogSession: + sid = str(uuid.uuid4())[:8] + session = DialogSession(sid, language, enable_vad) + self.sessions[sid] = session + log.info("Created dialog session %s (lang=%s, vad=%s)", sid, language, enable_vad) + return session + + def get_session(self, session_id: str) -> Optional[DialogSession]: + return self.sessions.get(session_id) + + def remove_session(self, session_id: str) -> bool: + if session_id in self.sessions: + del self.sessions[session_id] + log.info("Removed dialog session %s", session_id) + return True + return False + + +# ── gRPC Servicers ──────────────────────────────────────────────────────── +# These classes implement the gRPC service interfaces defined in +# speech_service.proto. Each servicer wraps one or more backend engines. + +class SpeechAsrServicer(spb_grpc.SpeechAsrServicer): + """ASR gRPC servicer — handles Recognize and RecognizeStream RPCs. + + Delegates to: + - asr_backend (WhisperASRBackend) for one-shot Recognize + - stream_asr_backend (FunASRStreamingBackend) for streaming RecognizeStream + + Audio adaptation: + Both RPCs call audio_utils.adapt_audio() to normalize input to + 16kHz mono pcm_s16le before passing to the backend, regardless + of the caller's audio format. + """ + + def __init__(self, asr_backend, stream_asr_backend): + self.asr_backend = asr_backend + self.stream_asr_backend = stream_asr_backend + + def Recognize(self, request, context): + """Handle one-shot ASR: receive complete audio, return transcription. + + Flow: + 1. Extract audio params from request (with defaults) + 2. adapt_audio() → normalize to 16kHz mono pcm_s16le + 3. WhisperASRBackend.recognize() → {text, confidence} + 4. Return RecognizeResponse + + Args: + request: RecognizeRequest proto with audio_data, encoding, + sample_rate_hz, language, channels, bits_per_sample. + context: gRPC context. + + Returns: + RecognizeResponse with text + confidence, or error string. + """ + from speech_service.audio_utils import adapt_audio + + encoding = request.encoding or "pcm_s16le" + sample_rate = request.sample_rate_hz or 16000 + channels = request.channels or 1 + bits_per_sample = request.bits_per_sample or 16 + language = request.language + + try: + # Adapt to 16kHz mono pcm_s16le regardless of input format + audio_data, _ = adapt_audio( + request.audio_data, + encoding=encoding, + sample_rate=sample_rate, + channels=channels, + bits_per_sample=bits_per_sample, + ) + result = self.asr_backend.recognize(audio_data, "pcm_s16le", 16000, language) + return spb.RecognizeResponse( + text=result["text"], + confidence=result.get("confidence", 0.0), + error="", + ) + except Exception as e: + log.exception("ASR recognize failed") + return spb.RecognizeResponse(text="", confidence=0.0, error=str(e)) + + def RecognizeStream(self, request_iterator, context): + """Handle streaming ASR: receive audio chunks, yield partial/final results. + + Flow: + 1. Read config from first message (AudioConfig + language) + 2. For each chunk: adapt_audio() → recognize_chunk() with cache + 3. Yield PARTIAL results (event_type=0) as they arrive + 4. On stream close: flush with is_final=True → yield FINAL (event_type=1) + 5. On error: yield ERROR (event_type=2) + + The cache dict maintains Paraformer streaming state across chunks. + It is created per-call and discarded when the stream ends. + + Args: + request_iterator: Stream of RecognizeStreamRequest messages. + context: gRPC context. + + Yields: + RecognizeStreamResponse with event_type, text, confidence. + """ + from speech_service.audio_utils import adapt_audio, parse_audio_config + + cache = {} + language = "" + config = {"encoding": "pcm_s16le", "sample_rate": 16000, "channels": 1, "bits_per_sample": 16} + chunk_count = 0 + + try: + for req in request_iterator: + if req.audio_config and (req.audio_config.encoding or req.audio_config.sample_rate_hz): + config = parse_audio_config(req.audio_config) + if req.language: + language = req.language + + chunk_data = bytes(req.chunk.data) if req.chunk and req.chunk.data else None + if chunk_data is None: + continue + + chunk_count += 1 + + # Adapt each chunk to 16kHz mono pcm_s16le + adapted, _ = adapt_audio( + chunk_data, + encoding=config["encoding"], + sample_rate=config["sample_rate"], + channels=config["channels"], + bits_per_sample=config["bits_per_sample"], + ) + + results = self.stream_asr_backend.recognize_chunk( + adapted, cache, is_final=False, + encoding="pcm_s16le", sample_rate=16000, + ) + for r in results: + if r.get("text"): + yield spb.RecognizeStreamResponse( + event_type=0, text=r["text"], + confidence=r.get("confidence", 0.0), + language=language, + ) + + # Final flush + if chunk_count > 0: + final_results = self.stream_asr_backend.recognize_chunk( + b"", cache, is_final=True, + encoding="pcm_s16le", sample_rate=16000, + ) + for r in final_results: + if r.get("text"): + yield spb.RecognizeStreamResponse( + event_type=1, text=r["text"], + confidence=r.get("confidence", 0.0), + language=language, is_final=True, + ) + else: + yield spb.RecognizeStreamResponse( + event_type=2, error="No audio data received", + ) + except Exception as e: + log.exception("ASR stream recognize failed") + yield spb.RecognizeStreamResponse(event_type=2, error=str(e)) + + +class SpeechTtsServicer(spb_grpc.SpeechTtsServicer): + """TTS gRPC servicer — handles Synthesize and SynthesizeStream RPCs. + + Delegates to EdgeTTSBackend for audio generation. + Output format: MP3 at 24kHz (fixed by Edge TTS). + + Note: Edge TTS uses asyncio internally, but gRPC servicers are sync. + Synthesize uses asyncio.run(), SynthesizeStream creates a dedicated + event loop to avoid "cannot run from running loop" conflicts. + """ + + def __init__(self, tts_backend): + self.tts_backend = tts_backend + + def Synthesize(self, request, context): + """Handle one-shot TTS: receive text, return complete MP3 audio. + + Args: + request: SynthesizeRequest with text, language, voice, speed. + context: gRPC context. + + Returns: + SynthesizeResponse with audio_data (MP3), encoding="mp3", + sample_rate_hz=24000. + """ + text = request.text + voice = request.voice + speed = request.speed or 1.0 + + try: + audio_data = asyncio.run(self.tts_backend.synthesize(text, voice, speed)) + return spb.SynthesizeResponse( + audio_data=audio_data, + encoding="mp3", + sample_rate_hz=24000, + error="", + ) + except Exception as e: + log.exception("TTS synthesize failed") + return spb.SynthesizeResponse(audio_data=b"", error=str(e)) + + def SynthesizeStream(self, request, context): + """Handle streaming TTS: receive text, yield MP3 audio chunks. + + Uses a dedicated asyncio event loop (not the main one) to drive + the Edge TTS async generator. Each chunk is yielded as a + SynthesizeStreamResponse. The last chunk has is_final=True. + + Args: + request: SynthesizeStreamRequest with text, language, voice, speed. + context: gRPC context. + + Yields: + SynthesizeStreamResponse with audio_data (MP3 chunk), + encoding="mp3", sample_rate_hz=24000, is_final on last. + """ + text = request.text + voice = request.voice + speed = request.speed or 1.0 + + try: + loop = asyncio.new_event_loop() + async_gen = self.tts_backend.synthesize_stream(text, voice, speed) + while True: + try: + chunk_data = loop.run_until_complete(async_gen.__anext__()) + yield spb.SynthesizeStreamResponse( + audio_data=chunk_data, + encoding="mp3", + sample_rate_hz=24000, + is_final=False, + ) + except StopAsyncIteration: + yield spb.SynthesizeStreamResponse( + audio_data=b"", encoding="mp3", + sample_rate_hz=24000, is_final=True, + ) + break + loop.close() + except Exception as e: + log.exception("TTS stream synthesize failed") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + +class SpeechDialogServicer(spb_grpc.SpeechDialogServicer): + """Dialog gRPC servicer — handles StartDialog RPC. + + Creates a DialogSession and streams state updates to the client. + The session stays alive until the client disconnects, then is + automatically cleaned up. + + This is a foundation for future VAD + turn-taking logic. Currently + it only tracks session state without active audio processing. + """ + + def __init__(self, dialog_manager): + self.dialog_manager = dialog_manager + + def StartDialog(self, request, context): + language = request.language or "zh-CN" + enable_vad = request.enable_vad if hasattr(request, "enable_vad") else True + + session = self.dialog_manager.create_session(language, enable_vad) + + yield spb.StartDialogResponse( + session_id=session.session_id, + state=DialogSession.STATE_MAP["IDLE"], + error="", + ) + + try: + while context.is_active(): + time.sleep(1.0) + except Exception: + pass + finally: + self.dialog_manager.remove_session(session.session_id) + + +# ── Atlas registration (optional) ───────────────────────────────────────── +# Registers the speech service with the Robonix Atlas control plane so that +# other nodes can discover it. If Atlas is unavailable or the runtime protos +# are not installed, the service runs in standalone mode. + +def _register_with_atlas(port: int) -> None: + """Register with Atlas control plane. Non-fatal on failure. + + Registration flow: + 1. RegisterNode: node_id="com.robonix.services.speech", + namespace="robonix/srv/speech", kind="service" + 2. DeclareInterface × 5: one per RPC (asr, asr_stream, tts, + tts_stream, dialog), each with its gRPC mode. + 3. Atlas returns allocated_endpoint for each interface. + + If robonix_runtime_pb2 is not importable, or Atlas is unreachable, + logs a warning and continues. The service is fully functional without + Atlas — it just won't be discoverable by other nodes. + + Args: + port: The gRPC port the service is listening on. + """ + try: + import robonix_runtime_pb2 as pb + import robonix_runtime_pb2_grpc as pb_grpc + except ImportError: + log.warning("robonix_runtime_pb2 not found, skipping Atlas registration") + return + + atlas_addr = os.environ.get("ROBONIX_ATLAS", "localhost:50051") + log.info("Connecting to Atlas at %s ...", atlas_addr) + + try: + channel = grpc.insecure_channel(atlas_addr) + stub = pb_grpc.RobonixRuntimeStub(channel) + + node_id = "com.robonix.services.speech" + stub.RegisterNode(pb.RegisterNodeRequest( + node_id=node_id, + namespace="robonix/srv/speech", + kind="service", + )) + log.info("Registered node %s with Atlas", node_id) + + contracts = [ + ("robonix/srv/speech/asr", "asr", "rpc"), + ("robonix/srv/speech/asr_stream", "asr_stream", "rpc_bidirectional_stream"), + ("robonix/srv/speech/tts", "tts", "rpc"), + ("robonix/srv/speech/tts_stream", "tts_stream", "rpc_server_stream"), + ("robonix/srv/speech/dialog", "dialog", "rpc_server_stream"), + ] + for contract_id, name, mode in contracts: + meta = json.dumps({"transport": "grpc", "contract": {"idl_type": "protobuf", "mode": mode}}) + resp = stub.DeclareInterface(pb.DeclareInterfaceRequest( + node_id=node_id, name=name, + supported_transports=["grpc"], + metadata_json=meta, listen_port=port, + contract_id=contract_id, + )) + log.info("Declared %s -> endpoint=%s", contract_id, resp.allocated_endpoint) + + channel.close() + except grpc.RpcError as e: + log.warning("Atlas not available (%s), running standalone", e.code()) + + +# ── Main ────────────────────────────────────────────────────────────────── +# Entry point. Called via `python -m speech_service.service` or via the +# robonix_manifest.yaml start command: `exec python -m speech_service.service`. + +def main() -> None: + """Start the speech service. + + Startup sequence: + 1. Initialize backends (real or mock based on SPEECH_CI_MODE) + 2. Create gRPC server with thread pool (8 workers) + 3. Register all 3 servicers (ASR, TTS, Dialog) + 4. Bind to SPEECH_BIND_ADDR:SPEECH_PORT (default 0.0.0.0:0 = auto) + 5. Optionally register with Atlas control plane + 6. Block until server terminates + """ + log.info("Starting speech service (ci_mode=%s)", CI_MODE) + + # Initialize backends + if CI_MODE: + asr_backend = MockASRBackend() + asr_stream_backend = MockASRStreamingBackend() + tts_backend = MockTTSBackend() + else: + asr_backend = WhisperASRBackend() + asr_stream_backend = FunASRStreamingBackend() + tts_backend = EdgeTTSBackend() + + dialog_manager = DialogManager() + + # Start gRPC server + server = grpc.server(futures.ThreadPoolExecutor(max_workers=8)) + + # Register servicers + spb_grpc.add_SpeechAsrServicer_to_server( + SpeechAsrServicer(asr_backend, asr_stream_backend), server) + spb_grpc.add_SpeechTtsServicer_to_server( + SpeechTtsServicer(tts_backend), server) + spb_grpc.add_SpeechDialogServicer_to_server( + SpeechDialogServicer(dialog_manager), server) + + # Bind port + bind_addr = os.environ.get("SPEECH_BIND_ADDR", "0.0.0.0") + port_str = os.environ.get("SPEECH_PORT", "0") + port = server.add_insecure_port(f"{bind_addr}:{port_str}") + if port == 0: + log.error("Failed to bind gRPC server on %s:%s", bind_addr, port_str) + sys.exit(1) + + server.start() + log.info("Speech service gRPC listening on %s:%d", bind_addr, port) + + # Try Atlas registration (non-blocking, continues on failure) + _register_with_atlas(port) + + server.wait_for_termination() + + +if __name__ == "__main__": + main() diff --git a/rust/examples/run.sh b/rust/examples/run.sh index 91f5ba6d3..6c0b01928 100755 --- a/rust/examples/run.sh +++ b/rust/examples/run.sh @@ -42,8 +42,10 @@ export START_PILOT="${START_PILOT:-1}" export START_EXECUTOR="${START_EXECUTOR:-1}" export START_LIAISON="${START_LIAISON:-1}" export START_MEMSEARCH="${START_MEMSEARCH:-1}" +export START_SPEECH_SERVICE="${START_SPEECH_SERVICE:-1}" +export START_AUDIO_DRIVER="${START_AUDIO_DRIVER:-1}" -export PYTHONPATH="${PACKAGES}/vlm_service:${PACKAGES}/memsearch_service${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH="${PACKAGES}/vlm_service:${PACKAGES}/memsearch_service:${PACKAGES}/speech_service:${PACKAGES}/audio_driver${PYTHONPATH:+:$PYTHONPATH}" rbnx() { (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "$@") @@ -103,6 +105,8 @@ pkill -9 -f 'robonix-liaison' 2>/dev/null || true pkill -9 -f 'vlm_service.service' 2>/dev/null || true pkill -9 -f 'memsearch_service.service' 2>/dev/null || true pkill -9 -f 'tiago_bridge.node' 2>/dev/null || true +pkill -9 -f 'speech_service.service' 2>/dev/null || true +pkill -9 -f 'audio_driver.node' 2>/dev/null || true sleep 0.3 SIM_STACK_DIR="${PACKAGES}/tiago_sim_stack" @@ -156,6 +160,20 @@ if [ "$START_MEMSEARCH" = "1" ]; then sleep 1 fi +if [ "$START_AUDIO_DRIVER" = "1" ]; then + echo "[example] rbnx start audio_driver (background)..." + rbnx_validate_build "$PACKAGES/audio_driver" + (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "${RBNX_START_OPTS[@]}" -p "$PACKAGES/audio_driver" -n com.robonix.prm.audio) & + sleep 1 +fi + +if [ "$START_SPEECH_SERVICE" = "1" ]; then + echo "[example] rbnx start speech_service (background)..." + rbnx_validate_build "$PACKAGES/speech_service" + (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "${RBNX_START_OPTS[@]}" -p "$PACKAGES/speech_service" -n com.robonix.services.speech) & + sleep 1 +fi + if [ "$START_SIM_STACK" = "1" ]; then echo "[example] rbnx start tiago_sim_stack (background, docker compose)..." (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "${RBNX_START_OPTS[@]}" -p "$PACKAGES/tiago_sim_stack" -n com.robonix.prm.tiago) & From 16315ae1926f16562626e5ee5ea035dba75c4c19 Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Sun, 19 Apr 2026 10:24:02 +0000 Subject: [PATCH 2/7] fix: restore slam.proto/prm_base.proto to dev state, keep only speech additions in robonix_msg.proto The previous commit accidentally included unrelated deletions from slam.proto, prm_base.proto, and robonix_msg.proto that were pre-existing local changes. This restores those files to their dev state and only adds the new audio/speech messages (AudioChunk, AudioConfig, DialogEvent, DialogState, VadEvent) to robonix_msg.proto. --- .../robonix_proto/prm_base.proto | 59 +++++++++ .../robonix_proto/robonix_msg.proto | 116 +++++++++++------- .../robonix_proto/slam.proto | 29 +++++ 3 files changed, 163 insertions(+), 41 deletions(-) diff --git a/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto b/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto index 19fc74a12..f13e7c8ee 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/prm_base.proto @@ -4,6 +4,10 @@ syntax = "proto3"; package robonix.prm_base; +import "geometry_msgs.proto"; +import "sensor_msgs.proto"; +import "std_msgs.proto"; + message MoveCommand { double linear_x = 1; double linear_y = 2; @@ -13,6 +17,15 @@ message MoveCommand { double angular_z = 6; } +message RobotState { + robonix.std_msgs.Header header = 1; + robonix.geometry_msgs.PoseStamped base_pose = 2; + robonix.sensor_msgs.JointState joint_state = 3; + robonix.geometry_msgs.PoseStamped tcp_pose = 4; + double gripper_width = 5; + bool is_grasped = 6; +} + message CancelNavigation_Request { string goal_id = 1; } @@ -22,6 +35,28 @@ message CancelNavigation_Response { string message = 2; } +message ExecuteMoveCommand_Request { + MoveCommand command = 1; +} + +message ExecuteMoveCommand_Response { + robonix.std_msgs.String status = 1; +} + +message GetCameraImage_Request { +} + +message GetCameraImage_Response { + robonix.sensor_msgs.Image image = 1; +} + +message GetLaserScan_Request { +} + +message GetLaserScan_Response { + robonix.sensor_msgs.LaserScan scan = 1; +} + message GetNavigationStatus_Request { string goal_id = 1; } @@ -32,6 +67,30 @@ message GetNavigationStatus_Response { bool terminal = 3; } +message GetRobotState_Request { +} + +message GetRobotState_Response { + RobotState state = 1; +} + +message Navigate_Request { + robonix.geometry_msgs.PoseStamped goal = 1; +} + +message Navigate_Response { + bool accepted = 1; + string status_message = 2; +} + +message PerceptionDetect_Request { + robonix.sensor_msgs.Image image = 1; +} + +message PerceptionDetect_Response { + robonix.std_msgs.String results = 1; +} + message StreamMove_Request { MoveCommand cmd = 1; } diff --git a/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto b/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto index 1c6959a57..1d2ba8cb3 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/robonix_msg.proto @@ -4,19 +4,9 @@ syntax = "proto3"; package robonix.robonix_msg; -message AudioChunk { - uint64 timestamp_ns = 1; - bytes data = 2; - uint32 sequence = 3; - float duration_s = 4; -} - -message AudioConfig { - string encoding = 1; - uint32 sample_rate_hz = 2; - uint32 channels = 3; - uint32 bits_per_sample = 4; -} +import "geometry_msgs.proto"; +import "sensor_msgs.proto"; +import "std_msgs.proto"; message BoundingBox { double scale_x = 1; @@ -72,27 +62,6 @@ message CommandResult { int32 code = 3; } -message DialogEvent { - uint64 timestamp_ns = 1; - string session_id = 2; - uint32 event_type = 3; - VadEvent vad_event = 4; - string text = 5; - float confidence = 6; - string language = 7; - string response_text = 8; - AudioChunk audio_chunk = 9; - DialogState dialog_state = 10; - string error = 11; -} - -message DialogState { - string session_id = 1; - uint32 state = 2; - string language = 3; - string error = 4; -} - message Dict { repeated string keys = 1; repeated string values = 2; @@ -123,6 +92,16 @@ message Object { repeated FrameMapping frame_mapping = 6; } +message Occupancy3D { + robonix.std_msgs.Header header = 1; + uint32 width = 2; + uint32 height = 3; + uint32 depth = 4; + float resolution = 5; + robonix.geometry_msgs.Point origin = 6; + repeated int32 data = 7; +} + message Point3D { double x = 1; double y = 2; @@ -137,6 +116,11 @@ message PrimitiveInstance { string metadata = 5; } +message RGBD { + robonix.sensor_msgs.Image rgb = 1; + robonix.sensor_msgs.Image depth = 2; +} + message Relation { RelationType relation_type = 1; string target_entity_id = 2; @@ -191,13 +175,6 @@ message ToolSpec { string input_schema_json = 3; } -message VadEvent { - uint64 timestamp_ns = 1; - uint32 event_type = 2; - float confidence = 3; - float energy_level = 4; -} - message Waypoint { string name = 1; double x = 2; @@ -206,3 +183,60 @@ message Waypoint { string frame_id = 5; } +message ZeroCopyFrame { + robonix.std_msgs.Header header = 1; + uint64 buffer_handle_id = 2; + string shm_name = 3; + uint32 width = 4; + uint32 height = 5; + uint32 channels = 6; + uint32 stride = 7; + BufferFormat format = 8; + MemoryDomain memory_domain = 9; + uint64 data_size = 10; + uint64 sequence = 11; + uint64 hw_timestamp_ns = 12; + uint64 sync_token = 13; + bool gpu_pinned = 14; + uint32 pool_id = 15; +} + +message AudioChunk { + uint64 timestamp_ns = 1; + bytes data = 2; + uint32 sequence = 3; + float duration_s = 4; +} + +message AudioConfig { + string encoding = 1; + uint32 sample_rate_hz = 2; + uint32 channels = 3; + uint32 bits_per_sample = 4; +} + +message DialogEvent { + uint64 timestamp_ns = 1; + string session_id = 2; + uint32 event_type = 3; + string text = 4; + float confidence = 5; + string language = 6; + bool is_final = 7; + string error = 8; +} + +message DialogState { + string session_id = 1; + uint32 state = 2; + string language = 3; + bool vad_enabled = 4; + string error = 5; +} + +message VadEvent { + uint64 timestamp_ns = 1; + bool voice_detected = 2; + float energy = 3; +} + diff --git a/rust/crates/robonix-interfaces/robonix_proto/slam.proto b/rust/crates/robonix-interfaces/robonix_proto/slam.proto index 1b2695815..49a21af68 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/slam.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/slam.proto @@ -4,6 +4,26 @@ syntax = "proto3"; package robonix.slam; +import "geometry_msgs.proto"; +import "std_msgs.proto"; + +message SlamStatus { + robonix.std_msgs.Header header = 1; + string mode = 2; + bool odom_alive = 3; + float odom_hz = 4; + bool cloud_alive = 5; + string map_file = 6; + uint64 total_odom_frames = 7; +} + +message GetSlamStatus_Request { +} + +message GetSlamStatus_Response { + SlamStatus status = 1; +} + message LoadMap_Request { string path = 1; } @@ -23,6 +43,15 @@ message SaveMap_Response { string message = 3; } +message SetInitialPose_Request { + robonix.geometry_msgs.PoseStamped pose = 1; +} + +message SetInitialPose_Response { + bool success = 1; + string message = 2; +} + message SwitchMode_Request { string mode = 1; } From 57d07526c077390902846d52be9b6b10941a9393 Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Sun, 19 Apr 2026 11:12:11 +0000 Subject: [PATCH 3/7] fix(speech): remove hardcoded local model path, use HuggingFace model ID as default Replace hardcoded /home/zzb/... path with openai/whisper-large-v3 so the model auto-downloads on first run via HuggingFace Hub. --- .../packages/speech_service/speech_service/service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/examples/packages/speech_service/speech_service/service.py b/rust/examples/packages/speech_service/speech_service/service.py index d07115c94..62a8ba6a6 100644 --- a/rust/examples/packages/speech_service/speech_service/service.py +++ b/rust/examples/packages/speech_service/speech_service/service.py @@ -110,10 +110,7 @@ def __init__(self): import torch from transformers import pipeline as hf_pipeline - model_path = os.environ.get( - "ASR_MODEL", - "/home/zzb/zzb_HDD/Project/LLM_for_ASR/model/whisper-large-merged", - ) + model_path = os.environ.get("ASR_MODEL", "openai/whisper-large-v3") device = os.environ.get("ASR_DEVICE", "cuda") chunk_length = float(os.environ.get("ASR_CHUNK_LENGTH", "30.0")) batch_size = int(os.environ.get("ASR_BATCH_SIZE", "4")) From 399676e16e33d98d01fd25d021da43416cda7dc0 Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Mon, 20 Apr 2026 07:56:42 +0000 Subject: [PATCH 4/7] =?UTF-8?q?fix(speech):=20graceful=20backend=20init=20?= =?UTF-8?q?failure=20=E2=80=94=20report=20errors,=20don't=20crash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each backend (Whisper, FunASR, Edge TTS) is now initialized independently with try/except: - Failure → clear boxed error message with actionable fix options (download model, set env var, or use CI mode) - Service still starts with whatever backends ARE available - Unavailable backends return gRPC UNAVAILABLE with fix instructions - If ALL backends fail, exit(1) with guidance No silent degradation — callers get explicit errors telling them exactly what's missing and how to fix it. --- .../speech_service/speech_service/service.py | 176 +++++++++++------- 1 file changed, 113 insertions(+), 63 deletions(-) diff --git a/rust/examples/packages/speech_service/speech_service/service.py b/rust/examples/packages/speech_service/speech_service/service.py index 62a8ba6a6..5528b16d6 100644 --- a/rust/examples/packages/speech_service/speech_service/service.py +++ b/rust/examples/packages/speech_service/speech_service/service.py @@ -454,10 +454,8 @@ class SpeechAsrServicer(spb_grpc.SpeechAsrServicer): - asr_backend (WhisperASRBackend) for one-shot Recognize - stream_asr_backend (FunASRStreamingBackend) for streaming RecognizeStream - Audio adaptation: - Both RPCs call audio_utils.adapt_audio() to normalize input to - 16kHz mono pcm_s16le before passing to the backend, regardless - of the caller's audio format. + If a backend is None (failed to load at startup), the corresponding RPC + returns a clear error with actionable instructions instead of crashing. """ def __init__(self, asr_backend, stream_asr_backend): @@ -465,22 +463,18 @@ def __init__(self, asr_backend, stream_asr_backend): self.stream_asr_backend = stream_asr_backend def Recognize(self, request, context): - """Handle one-shot ASR: receive complete audio, return transcription. - - Flow: - 1. Extract audio params from request (with defaults) - 2. adapt_audio() → normalize to 16kHz mono pcm_s16le - 3. WhisperASRBackend.recognize() → {text, confidence} - 4. Return RecognizeResponse - - Args: - request: RecognizeRequest proto with audio_data, encoding, - sample_rate_hz, language, channels, bits_per_sample. - context: gRPC context. + """Handle one-shot ASR: receive complete audio, return transcription.""" + if self.asr_backend is None: + context.set_code(grpc.StatusCode.UNAVAILABLE) + context.set_details( + "ASR backend not available. " + "Set ASR_MODEL to a local model path, or ensure network access " + "for auto-download from HuggingFace Hub. " + "Current ASR_MODEL=" + + os.environ.get("ASR_MODEL", "openai/whisper-large-v3") + ) + return spb.RecognizeResponse() - Returns: - RecognizeResponse with text + confidence, or error string. - """ from speech_service.audio_utils import adapt_audio encoding = request.encoding or "pcm_s16le" @@ -509,25 +503,18 @@ def Recognize(self, request, context): return spb.RecognizeResponse(text="", confidence=0.0, error=str(e)) def RecognizeStream(self, request_iterator, context): - """Handle streaming ASR: receive audio chunks, yield partial/final results. - - Flow: - 1. Read config from first message (AudioConfig + language) - 2. For each chunk: adapt_audio() → recognize_chunk() with cache - 3. Yield PARTIAL results (event_type=0) as they arrive - 4. On stream close: flush with is_final=True → yield FINAL (event_type=1) - 5. On error: yield ERROR (event_type=2) - - The cache dict maintains Paraformer streaming state across chunks. - It is created per-call and discarded when the stream ends. - - Args: - request_iterator: Stream of RecognizeStreamRequest messages. - context: gRPC context. + """Handle streaming ASR: receive audio chunks, yield partial/final results.""" + if self.stream_asr_backend is None: + context.set_code(grpc.StatusCode.UNAVAILABLE) + context.set_details( + "Streaming ASR backend not available. " + "Set FUNASR_MODEL to a local model path, or ensure network access " + "for auto-download. " + "Current FUNASR_MODEL=" + + os.environ.get("FUNASR_MODEL", "paraformer-zh-streaming") + ) + return - Yields: - RecognizeStreamResponse with event_type, text, confidence. - """ from speech_service.audio_utils import adapt_audio, parse_audio_config cache = {} @@ -606,16 +593,16 @@ def __init__(self, tts_backend): self.tts_backend = tts_backend def Synthesize(self, request, context): - """Handle one-shot TTS: receive text, return complete MP3 audio. - - Args: - request: SynthesizeRequest with text, language, voice, speed. - context: gRPC context. + """Handle one-shot TTS: receive text, return complete MP3 audio.""" + if self.tts_backend is None: + context.set_code(grpc.StatusCode.UNAVAILABLE) + context.set_details( + "TTS backend not available. " + "Edge TTS requires network access to Microsoft Cognitive Services. " + "Check your network connection or set SPEECH_CI_MODE=1 for testing." + ) + return spb.SynthesizeResponse() - Returns: - SynthesizeResponse with audio_data (MP3), encoding="mp3", - sample_rate_hz=24000. - """ text = request.text voice = request.voice speed = request.speed or 1.0 @@ -633,20 +620,15 @@ def Synthesize(self, request, context): return spb.SynthesizeResponse(audio_data=b"", error=str(e)) def SynthesizeStream(self, request, context): - """Handle streaming TTS: receive text, yield MP3 audio chunks. - - Uses a dedicated asyncio event loop (not the main one) to drive - the Edge TTS async generator. Each chunk is yielded as a - SynthesizeStreamResponse. The last chunk has is_final=True. - - Args: - request: SynthesizeStreamRequest with text, language, voice, speed. - context: gRPC context. + """Handle streaming TTS: receive text, yield MP3 audio chunks.""" + if self.tts_backend is None: + context.set_code(grpc.StatusCode.UNAVAILABLE) + context.set_details( + "TTS backend not available. " + "Edge TTS requires network access to Microsoft Cognitive Services." + ) + return - Yields: - SynthesizeStreamResponse with audio_data (MP3 chunk), - encoding="mp3", sample_rate_hz=24000, is_final on last. - """ text = request.text voice = request.voice speed = request.speed or 1.0 @@ -786,6 +768,9 @@ def main() -> None: Startup sequence: 1. Initialize backends (real or mock based on SPEECH_CI_MODE) + - Each backend init is wrapped in try/except + - Failed backends are set to None; servicers return UNAVAILABLE + - If ALL backends fail, exit with error 2. Create gRPC server with thread pool (8 workers) 3. Register all 3 servicers (ASR, TTS, Dialog) 4. Bind to SPEECH_BIND_ADDR:SPEECH_PORT (default 0.0.0.0:0 = auto) @@ -794,15 +779,80 @@ def main() -> None: """ log.info("Starting speech service (ci_mode=%s)", CI_MODE) - # Initialize backends + # ── Initialize backends ────────────────────────────────────────────── + asr_backend = None + asr_stream_backend = None + tts_backend = None + if CI_MODE: asr_backend = MockASRBackend() asr_stream_backend = MockASRStreamingBackend() tts_backend = MockTTSBackend() else: - asr_backend = WhisperASRBackend() - asr_stream_backend = FunASRStreamingBackend() - tts_backend = EdgeTTSBackend() + # Whisper ASR (one-shot) + try: + asr_backend = WhisperASRBackend() + except Exception as e: + log.error("╔══════════════════════════════════════════════════════════════╗") + log.error("║ Whisper ASR backend FAILED to initialize ║") + log.error("║ Error: %s", str(e)[:51]) + log.error("║ ║") + log.error("║ To fix, choose ONE of: ║") + log.error("║ 1. Ensure network access → auto-download from HuggingFace║") + log.error("║ 2. Set ASR_MODEL=/path/to/local/whisper/model ║") + log.error("║ 3. Set SPEECH_CI_MODE=1 for testing (mock backend) ║") + log.error("║ ║") + log.error("║ The service will start without ASR (Recognize → UNAVAILABLE║") + log.error("╚══════════════════════════════════════════════════════════════╝") + asr_backend = None + + # FunASR Paraformer (streaming ASR) + try: + asr_stream_backend = FunASRStreamingBackend() + except Exception as e: + log.error("╔══════════════════════════════════════════════════════════════╗") + log.error("║ FunASR streaming ASR backend FAILED to initialize ║") + log.error("║ Error: %s", str(e)[:51]) + log.error("║ ║") + log.error("║ To fix, choose ONE of: ║") + log.error("║ 1. pip install funasr + network access ║") + log.error("║ 2. Set FUNASR_MODEL=/path/to/local/paraformer/model ║") + log.error("║ 3. Set SPEECH_CI_MODE=1 for testing ║") + log.error("║ ║") + log.error("║ Streaming ASR (RecognizeStream) will be UNAVAILABLE ║") + log.error("╚══════════════════════════════════════════════════════════════╝") + asr_stream_backend = None + + # Edge TTS + try: + tts_backend = EdgeTTSBackend() + except Exception as e: + log.error("╔══════════════════════════════════════════════════════════════╗") + log.error("║ Edge TTS backend FAILED to initialize ║") + log.error("║ Error: %s", str(e)[:51]) + log.error("║ ║") + log.error("║ To fix: ensure network access to Microsoft TTS endpoint ║") + log.error("║ or set SPEECH_CI_MODE=1 for testing ║") + log.error("║ ║") + log.error("║ TTS (Synthesize/SynthesizeStream) will be UNAVAILABLE ║") + log.error("╚══════════════════════════════════════════════════════════════╝") + tts_backend = None + + # Bail out if nothing is available + available = [b is not None for b in (asr_backend, asr_stream_backend, tts_backend)] + if not any(available): + log.error("All backends failed — cannot start service.") + log.error("Set SPEECH_CI_MODE=1 for testing, or fix model/network issues above.") + sys.exit(1) + + # ── Startup summary ────────────────────────────────────────────────── + log.info("Backend status:") + log.info(" Whisper ASR (Recognize): %s", + "OK" if asr_backend is not None else "UNAVAILABLE") + log.info(" FunASR (RecognizeStream): %s", + "OK" if asr_stream_backend is not None else "UNAVAILABLE") + log.info(" Edge TTS (Synthesize): %s", + "OK" if tts_backend is not None else "UNAVAILABLE") dialog_manager = DialogManager() From 1ad501cc16ef15508accde9f853c1cfa320e01e2 Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Fri, 24 Apr 2026 03:59:24 +0000 Subject: [PATCH 5/7] fix(clippy): use sort_by_key instead of sort_by in robonix-pilot/vlm.rs --- rust/crates/robonix-pilot/src/vlm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/robonix-pilot/src/vlm.rs b/rust/crates/robonix-pilot/src/vlm.rs index ac2368107..38c96c1bd 100644 --- a/rust/crates/robonix-pilot/src/vlm.rs +++ b/rust/crates/robonix-pilot/src/vlm.rs @@ -294,7 +294,7 @@ impl VlmClient { }; if nodes.len() > 1 { - nodes.sort_by(|a, b| b.namespace.len().cmp(&a.namespace.len())); + nodes.sort_by_key(|b| std::cmp::Reverse(b.namespace.len())); log::warn!( "multiple VLM candidates ({}); using most specific namespace {:?}", nodes.len(), From 95a48e00814c4686a972c269adff6151604d47dd Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Fri, 24 Apr 2026 09:36:27 +0000 Subject: [PATCH 6/7] =?UTF-8?q?refactor(speech):=20address=20PR=20review?= =?UTF-8?q?=20=E2=80=94=20codegen,=20model=20download,=20English=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Switch to rbnx codegen for proto stubs - Delete hand-written speech_service.proto / audio_driver.proto - Keep speech_service.proto as docs/speech_service.proto.ref - build.sh now only calls `rbnx codegen`, no grpc_tools.protoc - service.py / node.py import from codegen-generated stubs (robonix_contracts_pb2_grpc, asr_pb2, tts_pb2, etc.) 2. Fix ROS IDL .srv files - Move --- separator to correct position (request fields before ---) - This fixes empty Request messages in generated protos 3. Add rpc_bidirectional_stream to codegen - New mode supported in contract_gen.rs - Uses stream_request / stream_response from contract TOML - Generates: rpc Stream(stream InType) returns (stream OutType) 4. Add speech/audio services to robonix_contracts.proto - SrvSpeechAsr, SrvSpeechAsrStream, SrvSpeechTts, SrvSpeechTtsStream, SrvSpeechDialog - PrmAudioMic, PrmAudioSpeaker 5. Model download at build time, not runtime - build.sh downloads Whisper + FunASR models - service.py uses local_files_only=True — fail fast if missing - No silent network downloads on boot 6. Translate all Chinese to English - audio_driver/README.md, speech_service/README.md - All code comments in English --- .../src/codegen/contract_gen.rs | 50 ++ .../lib/asr/srv/Recognize.srv | 3 +- .../lib/speech/srv/StartDialog.srv | 7 +- .../lib/tts/srv/Synthesize.srv | 5 +- .../lib/tts/srv/SynthesizeStream.srv | 3 +- .../robonix_proto/asr.proto | 13 +- .../robonix_proto/robonix_contracts.proto | 39 ++ .../robonix_proto/speech.proto | 11 +- .../robonix_proto/tts.proto | 19 +- rust/examples/packages/audio_driver/README.md | 210 ++++----- .../audio_driver/audio_driver/node.py | 129 ++---- .../audio_driver/proto/audio_driver.proto | 93 ---- .../packages/audio_driver/scripts/build.sh | 14 +- .../packages/speech_service/README.md | 124 ++--- .../speech_service.proto.ref} | 0 .../packages/speech_service/scripts/build.sh | 28 +- .../speech_service/speech_service/service.py | 435 +++++++++++------- 17 files changed, 606 insertions(+), 577 deletions(-) delete mode 100644 rust/examples/packages/audio_driver/proto/audio_driver.proto rename rust/examples/packages/speech_service/{proto/speech_service.proto => docs/speech_service.proto.ref} (100%) diff --git a/rust/crates/robonix-codegen/src/codegen/contract_gen.rs b/rust/crates/robonix-codegen/src/codegen/contract_gen.rs index a0d6977e3..5054e7647 100644 --- a/rust/crates/robonix-codegen/src/codegen/contract_gen.rs +++ b/rust/crates/robonix-codegen/src/codegen/contract_gen.rs @@ -47,6 +47,10 @@ struct IoMsgIo { #[derive(Debug, Deserialize)] struct IoSrvIo { srv: String, + #[serde(default)] + stream_request: Option, + #[serde(default)] + stream_response: Option, } #[derive(Debug, Deserialize)] @@ -158,6 +162,7 @@ pub fn generate( "rpc" => format_unary(in_t, out_t), "rpc_server_stream" | "topic_out" => format_stream_out(in_t, out_t), "rpc_client_stream" | "topic_in" => format_stream_in(in_t, out_t), + "rpc_bidirectional_stream" => format_bidi_stream(in_t, out_t), other => bail!( "unknown [mode].type '{other}' in contract {} (expected rpc | rpc_server_stream | rpc_client_stream | topic_out | topic_in)", c.contract.id @@ -205,6 +210,14 @@ fn format_stream_in(input: &ResolvedType, output: &ResolvedType) -> String { ) } +fn format_bidi_stream(input: &ResolvedType, output: &ResolvedType) -> String { + format!( + "rpc Stream(stream {}) returns (stream {});", + stream_element(input), + stream_element(output) + ) +} + fn format_unary(input: &ResolvedType, output: &ResolvedType) -> String { format!( "rpc Call({}) returns ({});", @@ -301,6 +314,15 @@ fn resolve_contract_io( })?; resolve_srv_client_stream(s, &c.contract.id, resolver, imports, needs_string_wire) } + "rpc_bidirectional_stream" => { + let s = s.ok_or_else(|| { + anyhow::anyhow!( + "contract {}: [mode].type = \"rpc_bidirectional_stream\" requires [io.srv]", + c.contract.id + ) + })?; + resolve_srv_bidi_stream(s, &c.contract.id, resolver, imports, needs_string_wire) + } "topic_out" => { let m = m.ok_or_else(|| { anyhow::anyhow!( @@ -407,6 +429,34 @@ fn resolve_srv_client_stream( Ok((in_t, out_t)) } +/// Bidirectional stream: uses `stream_request` and `stream_response` from the +/// contract TOML as stream element types. Falls back to deriving from the .srv +/// fields if those hints are absent. +fn resolve_srv_bidi_stream( + io: &IoSrvIo, + contract_id: &str, + _resolver: &mut MsgResolver, + imports: &mut BTreeSet, + _needs_string_wire: &mut bool, +) -> Result<(ResolvedType, ResolvedType)> { + let in_ref = io + .stream_request + .as_deref() + .ok_or_else(|| anyhow::anyhow!( + "contract {contract_id}: [mode] rpc_bidirectional_stream requires [io.srv].stream_request" + ))?; + let out_ref = io + .stream_response + .as_deref() + .ok_or_else(|| anyhow::anyhow!( + "contract {contract_id}: [mode] rpc_bidirectional_stream requires [io.srv].stream_response" + ))?; + + let in_t = resolve_io(in_ref, _resolver, imports, _needs_string_wire)?; + let out_t = resolve_io(out_ref, _resolver, imports, _needs_string_wire)?; + Ok((in_t, out_t)) +} + fn srv_request_to_contract_input( srv: &SrvSpec, resolver: &mut MsgResolver, diff --git a/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv b/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv index 17b8adedc..0f286e117 100644 --- a/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv +++ b/rust/crates/robonix-interfaces/lib/asr/srv/Recognize.srv @@ -1,12 +1,11 @@ # ASR one-shot recognition # abstract_interface_id: robonix/sys/speech/asr/recognize # Caller sends raw audio bytes; service returns a transcript string. ---- uint8[] audio_data string encoding # e.g. "pcm_s16le", "opus", "mp3" uint32 sample_rate_hz # e.g. 16000 string language # BCP-47, e.g. "zh-CN", "en-US", "" = auto-detect -=== +--- string text # recognised transcript (empty if silent / error) float32 confidence # 0.0 – 1.0; 0.0 if unavailable string error # non-empty on failure diff --git a/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv b/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv index 713fd1729..2704afec3 100644 --- a/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv +++ b/rust/crates/robonix-interfaces/lib/speech/srv/StartDialog.srv @@ -1,10 +1,7 @@ # Start a voice dialog session. # Returns initial dialog state; events are streamed via the Dialog contract. ---- -robonix_msg/msg/AudioConfig audio_config string language bool enable_vad +robonix_msg/msg/AudioConfig audio_config --- -string session_id -robonix_msg/msg/DialogState state -string error +robonix_msg/msg/DialogEvent event diff --git a/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv b/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv index ab4c1623c..23c0fb1d3 100644 --- a/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv +++ b/rust/crates/robonix-interfaces/lib/tts/srv/Synthesize.srv @@ -1,12 +1,11 @@ # TTS one-shot synthesis # abstract_interface_id: robonix/sys/speech/tts/synthesize ---- string text # text to synthesize string language # BCP-47, e.g. "zh-CN", "" = default string voice # voice ID, "" = default float32 speed # 1.0 = normal; < 1.0 = slower; > 1.0 = faster -=== -uint8[] audio_data # synthesized audio (format specified in encoding) +--- +uint8[] audio_data # synthesized audio (format specified in encoding) string encoding # e.g. "pcm_s16le", "mp3" uint32 sample_rate_hz string error # non-empty on failure diff --git a/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv b/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv index 3fffe4f3c..8b679b026 100644 --- a/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv +++ b/rust/crates/robonix-interfaces/lib/tts/srv/SynthesizeStream.srv @@ -1,11 +1,10 @@ # TTS streaming synthesis (server stream). # Client sends text request; server streams back SynthesizeAudioChunk. # Stream element type is referenced in the contract TOML. ---- string text string language string voice float32 speed robonix_msg/AudioConfig audio_config -=== +--- tts/SynthesizeAudioChunk chunk diff --git a/rust/crates/robonix-interfaces/robonix_proto/asr.proto b/rust/crates/robonix-interfaces/robonix_proto/asr.proto index 4e3ad2827..3fd4321f8 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/asr.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/asr.proto @@ -20,16 +20,16 @@ message RecognizeStreamEvent { } message Recognize_Request { -} - -message Recognize_Response { bytes audio_data = 1; string encoding = 2; uint32 sample_rate_hz = 3; string language = 4; - string text = 5; - float confidence = 6; - string error = 7; +} + +message Recognize_Response { + string text = 1; + float confidence = 2; + string error = 3; } message RecognizeStream_Request { @@ -40,4 +40,3 @@ message RecognizeStream_Request { message RecognizeStream_Response { RecognizeStreamEvent event = 1; } - diff --git a/rust/crates/robonix-interfaces/robonix_proto/robonix_contracts.proto b/rust/crates/robonix-interfaces/robonix_proto/robonix_contracts.proto index 41ccd0d3e..11797cec4 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/robonix_contracts.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/robonix_contracts.proto @@ -15,6 +15,10 @@ import "prm_base.proto"; import "sensor_msgs.proto"; import "slam.proto"; import "vlm.proto"; +import "asr.proto"; +import "tts.proto"; +import "speech.proto"; +import "robonix_msg.proto"; // contract: robonix/prm/base/cmd (v1) service PrmBaseCmd { @@ -181,3 +185,38 @@ service SrvSlamSwitchMode { rpc Call(robonix.slam.SwitchMode_Request) returns (robonix.slam.SwitchMode_Response); } +// contract: robonix/prm/audio/mic (v1) +service PrmAudioMic { + rpc Stream(google.protobuf.Empty) returns (stream robonix.robonix_msg.AudioChunk); +} + +// contract: robonix/prm/audio/speaker (v1) +service PrmAudioSpeaker { + rpc Stream(stream robonix.robonix_msg.AudioChunk) returns (google.protobuf.Empty); +} + +// contract: robonix/srv/speech/asr (v1) +service SrvSpeechAsr { + rpc Call(robonix.asr.Recognize_Request) returns (robonix.asr.Recognize_Response); +} + +// contract: robonix/srv/speech/asr_stream (v1) +service SrvSpeechAsrStream { + rpc Stream(stream robonix.asr.AsrAudioChunk) returns (stream robonix.asr.RecognizeStreamEvent); +} + +// contract: robonix/srv/speech/tts (v1) +service SrvSpeechTts { + rpc Call(robonix.tts.Synthesize_Request) returns (robonix.tts.Synthesize_Response); +} + +// contract: robonix/srv/speech/tts_stream (v1) +service SrvSpeechTtsStream { + rpc Stream(robonix.tts.SynthesizeStream_Request) returns (stream robonix.tts.SynthesizeAudioChunk); +} + +// contract: robonix/srv/speech/dialog (v1) +service SrvSpeechDialog { + rpc Stream(robonix.speech.StartDialog_Request) returns (stream robonix.robonix_msg.DialogEvent); +} + diff --git a/rust/crates/robonix-interfaces/robonix_proto/speech.proto b/rust/crates/robonix-interfaces/robonix_proto/speech.proto index 4d49272aa..e57cd6ee4 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/speech.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/speech.proto @@ -16,14 +16,11 @@ message DialogRequest { } message StartDialog_Request { + string language = 1; + bool enable_vad = 2; + robonix.robonix_msg.AudioConfig audio_config = 3; } message StartDialog_Response { - robonix.robonix_msg.AudioConfig audio_config = 1; - string language = 2; - bool enable_vad = 3; - string session_id = 4; - robonix.robonix_msg.DialogState state = 5; - string error = 6; + robonix.robonix_msg.DialogEvent event = 1; } - diff --git a/rust/crates/robonix-interfaces/robonix_proto/tts.proto b/rust/crates/robonix-interfaces/robonix_proto/tts.proto index 5311a6914..42bf29ec5 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/tts.proto +++ b/rust/crates/robonix-interfaces/robonix_proto/tts.proto @@ -14,28 +14,27 @@ message SynthesizeAudioChunk { } message Synthesize_Request { -} - -message Synthesize_Response { string text = 1; string language = 2; string voice = 3; float speed = 4; - bytes audio_data = 5; - string encoding = 6; - uint32 sample_rate_hz = 7; - string error = 8; } -message SynthesizeStream_Request { +message Synthesize_Response { + bytes audio_data = 1; + string encoding = 2; + uint32 sample_rate_hz = 3; + string error = 4; } -message SynthesizeStream_Response { +message SynthesizeStream_Request { string text = 1; string language = 2; string voice = 3; float speed = 4; robonix.robonix_msg.AudioConfig audio_config = 5; - SynthesizeAudioChunk chunk = 6; } +message SynthesizeStream_Response { + SynthesizeAudioChunk chunk = 1; +} diff --git a/rust/examples/packages/audio_driver/README.md b/rust/examples/packages/audio_driver/README.md index 9c6796f09..4b319719c 100644 --- a/rust/examples/packages/audio_driver/README.md +++ b/rust/examples/packages/audio_driver/README.md @@ -1,130 +1,130 @@ # Audio Driver -音频驱动 — Robonix prm 层，自动扫描 ALSA 设备，提供麦克风采集和扬声器播放的 gRPC 流式接口。 +Audio driver -- Robonix prm layer, automatically scans ALSA devices, provides gRPC streaming interfaces for microphone capture and speaker playback. -**本包同时作为所有未来 prm driver 包的参考模板。** +**This package also serves as a reference template for all future prm driver packages.** -## 架构定位 +## Architecture Position ``` ┌─────────────────────────────────────────────┐ │ speech_service (robonix/srv/speech) │ │ ↕ gRPC: AudioChunk │ ├─────────────────────────────────────────────┤ -│ audio_driver ← 你在这里 (robonix/prm/audio) │ +│ audio_driver ← you are here (robonix/prm/audio) │ │ ↕ ALSA: arecord / aplay │ ├─────────────────────────────────────────────┤ │ Hardware (USB Mic / Speaker) │ └─────────────────────────────────────────────┘ ``` -- **上层**：通过 gRPC 向 speech_service 等服务提供音频流 -- **下层**：通过 ALSA 子系统直接操控声卡硬件 -- **自动化**：启动时自动扫描 USB 麦克风/扬声器，无需手动配置 +- **Upper layer**: Provides audio streams to services like speech_service via gRPC +- **Lower layer**: Directly controls sound card hardware through the ALSA subsystem +- **Automation**: Automatically scans USB microphones/speakers at startup, no manual configuration needed -## gRPC 接口 +## gRPC Interfaces -| 服务 | RPC | 模式 | Contract ID | 数据流 | -|------|-----|------|-------------|--------| -| PrmAudioMic | Stream | Server-stream | `robonix/prm/audio/mic` | 麦克风 → 调用方 | -| PrmAudioSpeaker | Stream | Client-stream | `robonix/prm/audio/speaker` | 调用方 → 扬声器 | +| Service | RPC | Mode | Contract ID | Data Flow | +|---------|-----|------|-------------|-----------| +| PrmAudioMic | Stream | Server-stream | `robonix/prm/audio/mic` | Microphone -> Caller | +| PrmAudioSpeaker | Stream | Client-stream | `robonix/prm/audio/speaker` | Caller -> Speaker | -## 目录结构 +## Directory Structure ``` audio_driver/ ├── proto/ -│ └── audio_driver.proto # 自包含 gRPC 定义 -├── proto_gen/ # build 时生成的 *_pb2.py（git 忽略） +│ └── audio_driver.proto # Self-contained gRPC definitions +├── proto_gen/ # Generated *_pb2.py at build time (git-ignored) ├── scripts/ -│ └── build.sh # proto 代码生成 +│ └── build.sh # Proto code generation ├── audio_driver/ -│ ├── __init__.py # 包入口 -│ ├── node.py # 主入口：Atlas 注册 + daemon 线程 + main() -│ ├── alsa_utils.py # ALSA 设备扫描 + 驱动注册表 -│ ├── mic_driver.py # 麦克风采集（arecord subprocess） -│ └── speaker_driver.py # 扬声器播放（aplay subprocess） -├── robonix_manifest.yaml # Robonix 包描述 +│ ├── __init__.py # Package entry point +│ ├── node.py # Main entry: Atlas registration + daemon threads + main() +│ ├── alsa_utils.py # ALSA device scanning + driver registry +│ ├── mic_driver.py # Microphone capture (arecord subprocess) +│ └── speaker_driver.py # Speaker playback (aplay subprocess) +├── robonix_manifest.yaml # Robonix package descriptor ├── requirements.txt └── .gitignore ``` -## 快速启动 +## Quick Start -### 1. 安装依赖 +### 1. Install Dependencies ```bash pip install -r requirements.txt -# 核心依赖：grpcio, grpcio-tools, protobuf -# 系统依赖：arecord, aplay（alsa-utils 包） +# Core dependencies: grpcio, grpcio-tools, protobuf +# System dependencies: arecord, aplay (alsa-utils package) ``` -### 2. 生成 proto stubs +### 2. Generate Proto Stubs ```bash bash scripts/build.sh ``` -### 3. 启动驱动 +### 3. Start the Driver ```bash -# 正常模式（自动扫描设备 + Atlas 注册） +# Normal mode (auto-scan devices + Atlas registration) export PYTHONPATH=$(pwd)/proto_gen:${PYTHONPATH:-} python -m audio_driver.node -# 独立模式（跳过 Atlas 注册） +# Standalone mode (skip Atlas registration) AUDIO_DRIVER_STANDALONE=1 python -m audio_driver.node -# 指定设备（覆盖自动检测） +# Specify devices (override auto-detection) AUDIO_MIC_DEVICE=hw:1,0 AUDIO_SPEAKER_DEVICE=hw:0,0 python -m audio_driver.node ``` -### 4. 通过 Robonix 启动 +### 4. Launch via Robonix ```bash rbnx run com.robonix.example.audio_driver ``` -## 环境变量 +## Environment Variables -| 变量 | 默认值 | 说明 | -|------|--------|------| -| `AUDIO_MIC_DEVICE` | 自动检测 | 麦克风 ALSA 设备（如 `hw:1,0`） | -| `AUDIO_MIC_SAMPLE_RATE` | `16000` | 采集采样率（Hz） | -| `AUDIO_MIC_CHANNELS` | `1` | 采集声道数 | -| `AUDIO_MIC_BITS` | `16` | 采集位深 | -| `AUDIO_MIC_CHUNK_MS` | `100` | 每次 chunk 时长（毫秒） | -| `AUDIO_MIC_PORT` | `0`（自动分配） | Mic gRPC 端口 | -| `AUDIO_SPEAKER_DEVICE` | 自动检测 | 扬声器 ALSA 设备 | -| `AUDIO_SPEAKER_SAMPLE_RATE` | `24000` | 播放采样率（Hz） | -| `AUDIO_SPEAKER_CHANNELS` | `1` | 播放声道数 | -| `AUDIO_SPEAKER_BITS` | `16` | 播放位深 | -| `AUDIO_SPEAKER_PORT` | `0`（自动分配） | Speaker gRPC 端口 | -| `AUDIO_DRIVER_STANDALONE` | — | 设为 `1` 跳过 Atlas 注册 | -| `ROBONIX_ATLAS` | `localhost:50051` | Atlas 控制面地址 | -| `ROBONIX_NODE_ID` | `com.robonix.prm.audio` | Atlas 节点 ID | +| Variable | Default | Description | +|----------|---------|-------------| +| `AUDIO_MIC_DEVICE` | Auto-detect | Microphone ALSA device (e.g. `hw:1,0`) | +| `AUDIO_MIC_SAMPLE_RATE` | `16000` | Capture sample rate (Hz) | +| `AUDIO_MIC_CHANNELS` | `1` | Capture channel count | +| `AUDIO_MIC_BITS` | `16` | Capture bit depth | +| `AUDIO_MIC_CHUNK_MS` | `100` | Chunk duration per read (milliseconds) | +| `AUDIO_MIC_PORT` | `0` (auto-assign) | Mic gRPC port | +| `AUDIO_SPEAKER_DEVICE` | Auto-detect | Speaker ALSA device | +| `AUDIO_SPEAKER_SAMPLE_RATE` | `24000` | Playback sample rate (Hz) | +| `AUDIO_SPEAKER_CHANNELS` | `1` | Playback channel count | +| `AUDIO_SPEAKER_BITS` | `16` | Playback bit depth | +| `AUDIO_SPEAKER_PORT` | `0` (auto-assign) | Speaker gRPC port | +| `AUDIO_DRIVER_STANDALONE` | — | Set to `1` to skip Atlas registration | +| `ROBONIX_ATLAS` | `localhost:50051` | Atlas control plane address | +| `ROBONIX_NODE_ID` | `com.robonix.prm.audio` | Atlas node ID | -## 设备自动发现 +## Automatic Device Discovery -启动时执行以下流程： +At startup, the following sequence is executed: ``` -1. arecord -l → 解析输入设备（麦克风） -2. aplay -l → 解析输出设备（扬声器） -3. 合并去重 → 同一 card/device 的设备标记 is_input + is_output -4. 选择默认 → 优先 USB 设备（card >= 1），其次任意可用设备 -5. 环境变量覆盖 → 如果设置了 AUDIO_MIC_DEVICE / AUDIO_SPEAKER_DEVICE，使用指定值 +1. arecord -l → Parse input devices (microphones) +2. aplay -l → Parse output devices (speakers) +3. Merge & deduplicate → Same card/device marked as both is_input + is_output +4. Select defaults → Prefer USB devices (card >= 1), then any available device +5. Environment variable override → If AUDIO_MIC_DEVICE / AUDIO_SPEAKER_DEVICE is set, use specified value ``` -## 驱动注册表 +## Driver Registry -支持通过插件机制添加硬件专用驱动： +Hardware-specific drivers can be added via a plugin mechanism: ```python from audio_driver.alsa_utils import AudioDeviceDriver, register_driver class RespeakerDriver(AudioDeviceDriver): - """ReSpeaker 多通道麦克风阵列驱动""" + """ReSpeaker multi-channel microphone array driver""" def detect(self, devices): return [d for d in devices if "ReSpeaker" in d.name] @@ -135,71 +135,71 @@ class RespeakerDriver(AudioDeviceDriver): register_driver(RespeakerDriver()) ``` -内置驱动： -- **DefaultAlsaDriver** — 接受所有标准 ALSA 设备（默认注册） +Built-in drivers: +- **DefaultAlsaDriver** -- Accepts all standard ALSA devices (registered by default) -## 模块说明 +## Module Descriptions -### mic_driver.py — 麦克风采集 +### mic_driver.py -- Microphone Capture ``` -ALSA 设备 → arecord subprocess → stdout → MicDriver.read_chunk() → AudioChunk dict +ALSA device → arecord subprocess → stdout → MicDriver.read_chunk() → AudioChunk dict ``` -- 调用 `arecord -D hw:X,Y -f S16_LE -r 16000 -c 1 -t raw` -- `read_chunk()` 阻塞读取固定大小字节（默认 100ms = 3200 bytes @ 16kHz mono s16le） -- 返回 dict: `{timestamp_ns, data, sequence, duration_s}` +- Calls `arecord -D hw:X,Y -f S16_LE -r 16000 -c 1 -t raw` +- `read_chunk()` blocks reading a fixed-size byte count (default 100ms = 3200 bytes @ 16kHz mono s16le) +- Returns dict: `{timestamp_ns, data, sequence, duration_s}` -### speaker_driver.py — 扬声器播放 +### speaker_driver.py -- Speaker Playback ``` -gRPC AudioChunk → SpeakerDriver.play_chunk() → aplay stdin → ALSA 设备 +gRPC AudioChunk → SpeakerDriver.play_chunk() → aplay stdin → ALSA device ``` -- 调用 `aplay -D hw:X,Y -f S16_LE -r 24000 -c 1 -t raw` -- 惰性启动：首次 `play_chunk()` 时才启动 aplay -- 自动重启：BrokenPipeError 时自动重启 aplay 并重试 -- 线程安全：内部使用 threading.Lock +- Calls `aplay -D hw:X,Y -f S16_LE -r 24000 -c 1 -t raw` +- Lazy start: aplay is not started until the first `play_chunk()` call +- Auto-restart: On BrokenPipeError, aplay is automatically restarted and the chunk is retried +- Thread-safe: Uses threading.Lock internally -### node.py — 主入口 +### node.py -- Main Entry Point -启动流程（参照 tiago_bridge 模式）： +Startup sequence (following the tiago_bridge pattern): ``` -1. scan_alsa_devices() → 发现硬件 -2. find_default_mic/speaker() → 选择设备 -3. MicDriver / SpeakerDriver() → 创建驱动实例 -4. auto-pick ports → 自动分配 gRPC 端口 -5. RegisterNode → Atlas 注册 -6. DeclareInterface × 2 → 声明 mic + speaker 接口 -7. daemon threads → 启动心跳 + 2 个 gRPC server -8. main thread sleep → 阻塞等待 +1. scan_alsa_devices() → Discover hardware +2. find_default_mic/speaker() → Select devices +3. MicDriver / SpeakerDriver() → Create driver instances +4. auto-pick ports → Auto-assign gRPC ports +5. RegisterNode → Atlas registration +6. DeclareInterface × 2 → Declare mic + speaker interfaces +7. daemon threads → Start heartbeat + 2 gRPC servers +8. main thread sleep → Block and wait ``` -## 作为参考模板 +## As a Reference Template -新建 prm driver 时复制此包并修改： +When creating a new prm driver, copy this package and modify: -| 替换项 | 本包值 | 你的包 | -|--------|--------|--------| -| proto 消息类型 | AudioConfig, AudioChunk | 你的硬件数据类型 | -| gRPC 服务 | PrmAudioMic, PrmAudioSpeaker | 你的设备接口 | -| Contract ID | robonix/prm/audio/* | robonix/prm/你的设备/* | -| 扫描工具 | arecord -l / aplay -l | 你的设备发现方式 | -| 驱动类 | MicDriver, SpeakerDriver | 你的设备驱动 | -| 环境变量前缀 | AUDIO_ | 你的设备前缀 | -| manifest node ID | com.robonix.prm.audio | com.robonix.prm.你的设备 | +| Replace | This Package | Your Package | +|---------|-------------|-------------| +| Proto message types | AudioConfig, AudioChunk | Your hardware data types | +| gRPC services | PrmAudioMic, PrmAudioSpeaker | Your device interfaces | +| Contract IDs | robonix/prm/audio/* | robonix/prm/your_device/* | +| Scanning tools | arecord -l / aplay -l | Your device discovery method | +| Driver classes | MicDriver, SpeakerDriver | Your device drivers | +| Environment variable prefix | AUDIO_ | Your device prefix | +| Manifest node ID | com.robonix.prm.audio | com.robonix.prm.your_device | -保留不变的部分： -- `_ensure_proto_gen()` — proto stub 查找逻辑 -- `_register_with_atlas()` — Atlas 注册 + 心跳 -- daemon 线程模式 — 每个 interface 一个 gRPC server 线程 -- `robonix_manifest.yaml` 结构 -- `scripts/build.sh` proto 生成 +Keep unchanged: +- `_ensure_proto_gen()` -- Proto stub lookup logic +- `_register_with_atlas()` -- Atlas registration + heartbeat +- Daemon thread pattern -- One gRPC server thread per interface +- `robonix_manifest.yaml` structure +- `scripts/build.sh` proto generation -## Atlas 集成 +## Atlas Integration -- **RegisterNode**: `com.robonix.prm.audio`，namespace `robonix/prm/audio`，kind `primitive` -- **DeclareInterface**: mic（server-stream）+ speaker（client-stream） -- **心跳**: 每 15 秒发送 NodeHeartbeat -- **降级**: Atlas 不可用时自动独立运行 +- **RegisterNode**: `com.robonix.prm.audio`, namespace `robonix/prm/audio`, kind `primitive` +- **DeclareInterface**: mic (server-stream) + speaker (client-stream) +- **Heartbeat**: Sends NodeHeartbeat every 15 seconds +- **Degradation**: Automatically runs standalone when Atlas is unavailable diff --git a/rust/examples/packages/audio_driver/audio_driver/node.py b/rust/examples/packages/audio_driver/audio_driver/node.py index 96d2d61b7..42d80fa74 100644 --- a/rust/examples/packages/audio_driver/audio_driver/node.py +++ b/rust/examples/packages/audio_driver/audio_driver/node.py @@ -18,6 +18,10 @@ - Speaker gRPC server thread (PrmAudioSpeakerServicer) 5. Main thread blocks until KeyboardInterrupt +Proto stubs are loaded from codegen-generated files: + - robonix_contracts_pb2_grpc (servicer base classes + registration fns) + - robonix_msg_pb2 (AudioChunk message) + This package is designed as a **REFERENCE TEMPLATE** for all future prm driver packages. The same pattern should be replicated for camera, lidar, IMU, and other hardware drivers. @@ -52,14 +56,14 @@ # ── Proto stub resolution ────────────────────────────────────────────────── # Walks up from this file's directory looking for proto_gen/ containing the -# generated audio_driver_pb2.py. Adds it to sys.path so the proto imports work -# regardless of the current working directory. +# codegen-generated robonix_contracts_pb2.py. Adds it to sys.path so the +# proto imports work regardless of the current working directory. def _ensure_proto_gen() -> None: d = Path(__file__).resolve().parent while d.parent != d: pg = d / "proto_gen" - if pg.is_dir() and (pg / "audio_driver_pb2.py").exists(): + if pg.is_dir() and (pg / "robonix_contracts_pb2.py").exists(): sys.path.insert(0, str(pg)) return d = d.parent @@ -67,8 +71,8 @@ def _ensure_proto_gen() -> None: _ensure_proto_gen() import grpc -import audio_driver_pb2 as pb -import audio_driver_pb2_grpc as pb_grpc +import robonix_msg_pb2 +import robonix_contracts_pb2_grpc as contracts_grpc from audio_driver.alsa_utils import scan_alsa_devices, find_default_mic, find_default_speaker from audio_driver.mic_driver import MicDriver @@ -79,7 +83,7 @@ def _ensure_proto_gen() -> None: # the corresponding driver. The servicers are registered on separate gRPC # servers running in daemon threads. -class PrmAudioMicServicer(pb_grpc.PrmAudioMicServicer): +class MicServicer(contracts_grpc.PrmAudioMicServicer): """Implements contract robonix/prm/audio/mic — server-streaming mic audio. When a client calls Stream(): @@ -105,7 +109,7 @@ def Stream(self, request, context): chunk = self.mic_driver.read_chunk() if chunk is None: break - yield pb.AudioChunk( + yield robonix_msg_pb2.AudioChunk( timestamp_ns=chunk["timestamp_ns"], data=chunk["data"], sequence=chunk["sequence"], @@ -116,13 +120,12 @@ def Stream(self, request, context): log.info("Mic stream client disconnected") -class PrmAudioSpeakerServicer(pb_grpc.PrmAudioSpeakerServicer): +class SpeakerServicer(contracts_grpc.PrmAudioSpeakerServicer): """Implements contract robonix/prm/audio/speaker — client-streaming playback. - When a client calls Stream() with a stream of SpeakerChunkRequest: - 1. Reads audio_config from the first message (if present) and logs it - 2. For each chunk with data, calls speaker_driver.play_chunk() - 3. On stream end, returns SpeakerStreamResponse(ok=True) + When a client calls Stream() with a stream of AudioChunk messages: + 1. For each chunk with data, calls speaker_driver.play_chunk() + 2. On stream end, returns google.protobuf.Empty The SpeakerDriver handles lazy-starting aplay and auto-restart on errors. """ @@ -132,29 +135,20 @@ def __init__(self, speaker_driver: SpeakerDriver): def Stream(self, request_iterator, context): log.info("Speaker stream client connected") - config_set = False try: - for req in request_iterator: - if req.audio_config and not config_set: - log.info("Speaker config: rate=%d ch=%d bits=%d enc=%s", - req.audio_config.sample_rate_hz, - req.audio_config.channels, - req.audio_config.bits_per_sample, - req.audio_config.encoding) - config_set = True - - if req.chunk and req.chunk.data: - self.speaker_driver.play_chunk(bytes(req.chunk.data)) + for chunk in request_iterator: + if chunk.data: + self.speaker_driver.play_chunk(bytes(chunk.data)) finally: log.info("Speaker stream client disconnected") - return pb.SpeakerStreamResponse(ok=True) + return robonix_msg_pb2.Empty() # ── Atlas registration (optional) ───────────────────────────────────────── # Follows the tiago_bridge pattern exactly: # 1. RegisterNode — register as a primitive under robonix/prm/audio -# 2. DeclareInterface × 2 — declare mic and speaker endpoints +# 2. DeclareInterface x 2 — declare mic and speaker endpoints # 3. Heartbeat thread — send NodeHeartbeat every 15 seconds def _register_with_atlas(mic_port: int, speaker_port: int) -> None: @@ -239,48 +233,6 @@ def _heartbeat(): log.warning("Atlas not available (%s), running standalone", e.code()) -# ── gRPC server helpers ─────────────────────────────────────────────────── -# Each interface runs in its own daemon thread with a separate gRPC server. -# This isolates mic and speaker so a problem with one doesn't affect the other. - -def _run_mic_server(port: int, mic_driver: MicDriver) -> None: - """Start and run the mic gRPC server (blocks until terminated). - - Creates a gRPC server with 4 worker threads, registers the - PrmAudioMicServicer, binds to 0.0.0.0:, and blocks. - - Args: - port: TCP port to listen on. - mic_driver: MicDriver instance to stream audio from. - """ - server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) - pb_grpc.add_PrmAudioMicServicer_to_server( - PrmAudioMicServicer(mic_driver), server) - server.add_insecure_port(f"0.0.0.0:{port}") - server.start() - log.info("Mic gRPC server on port %d", port) - server.wait_for_termination() - - -def _run_speaker_server(port: int, speaker_driver: SpeakerDriver) -> None: - """Start and run the speaker gRPC server (blocks until terminated). - - Creates a gRPC server with 4 worker threads, registers the - PrmAudioSpeakerServicer, binds to 0.0.0.0:, and blocks. - - Args: - port: TCP port to listen on. - speaker_driver: SpeakerDriver instance to play audio through. - """ - server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) - pb_grpc.add_PrmAudioSpeakerServicer_to_server( - PrmAudioSpeakerServicer(speaker_driver), server) - server.add_insecure_port(f"0.0.0.0:{port}") - server.start() - log.info("Speaker gRPC server on port %d", port) - server.wait_for_termination() - - # ── Main ────────────────────────────────────────────────────────────────── # Entry point. Called via `python -m audio_driver.node` or via the # robonix_manifest.yaml start command. @@ -292,10 +244,10 @@ def main() -> None: 1. Discover ALSA devices via scan_alsa_devices() 2. Find mic and speaker devices (auto-detect or env override) 3. Create MicDriver and SpeakerDriver instances - 4. Auto-pick free ports for mic and speaker gRPC servers - 5. Optionally register with Atlas (RegisterNode + DeclareInterface) - 6. Start mic gRPC server in a daemon thread - 7. Start speaker gRPC server in a daemon thread + 4. Create a single gRPC server with both servicers + 5. Auto-pick free ports for mic and speaker + 6. Optionally register with Atlas (RegisterNode + DeclareInterface) + 7. Start the gRPC server 8. Block main thread until KeyboardInterrupt """ log.info("Starting audio driver") @@ -357,7 +309,6 @@ def main() -> None: speaker_port = int(os.environ.get("AUDIO_SPEAKER_PORT", "0")) if mic_port == 0: - # Auto-pick: bind temporary server to get a free port import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("0.0.0.0", 0)) @@ -376,18 +327,32 @@ def main() -> None: if not standalone: _register_with_atlas(mic_port, speaker_port) - # 5. Start gRPC servers + # 5. Start gRPC server(s) + # Mic and speaker run on separate servers in daemon threads so they are + # isolated — a problem with one does not affect the other. if mic_driver: - threading.Thread( - target=_run_mic_server, args=(mic_port, mic_driver), - daemon=True, name="mic-grpc", - ).start() + def _run_mic_server(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) + contracts_grpc.add_PrmAudioMicServicer_to_server( + MicServicer(mic_driver), server) + server.add_insecure_port(f"0.0.0.0:{mic_port}") + server.start() + log.info("Mic gRPC server on port %d", mic_port) + server.wait_for_termination() + + threading.Thread(target=_run_mic_server, daemon=True, name="mic-grpc").start() if speaker_driver: - threading.Thread( - target=_run_speaker_server, args=(speaker_port, speaker_driver), - daemon=True, name="speaker-grpc", - ).start() + def _run_speaker_server(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) + contracts_grpc.add_PrmAudioSpeakerServicer_to_server( + SpeakerServicer(speaker_driver), server) + server.add_insecure_port(f"0.0.0.0:{speaker_port}") + server.start() + log.info("Speaker gRPC server on port %d", speaker_port) + server.wait_for_termination() + + threading.Thread(target=_run_speaker_server, daemon=True, name="speaker-grpc").start() log.info("Audio driver ready (mic=%s, speaker=%s)", f"port {mic_port}" if mic_driver else "none", diff --git a/rust/examples/packages/audio_driver/proto/audio_driver.proto b/rust/examples/packages/audio_driver/proto/audio_driver.proto deleted file mode 100644 index f13d8f47f..000000000 --- a/rust/examples/packages/audio_driver/proto/audio_driver.proto +++ /dev/null @@ -1,93 +0,0 @@ -// ────────────────────────────────────────────────────────────────────────── -// Audio Driver — gRPC service definitions for mic capture & speaker playback -// -// Architecture position: robonix/prm/audio (primitive / hardware layer) -// - Auto-scans ALSA devices (arecord -l / aplay -l) -// - Captures raw PCM from microphone → streams AudioChunk upward -// - Receives AudioChunk from upper-layer services → plays via speaker -// - Registers with Atlas as primitive node under robonix/prm/audio -// -// This package serves as a REFERENCE TEMPLATE for all future prm driver -// packages. Key patterns to replicate: -// 1. Self-contained proto (inlined types, no cross-package imports) -// 2. ALSA auto-detection via subprocess (arecord/aplay) -// 3. Driver registry for hardware extensibility (AudioDeviceDriver ABC) -// 4. Atlas registration + heartbeat (tiago_bridge pattern) -// 5. Daemon threads for gRPC servers (one per interface) -// -// Build: scripts/build.sh generates *_pb2.py / *_pb2_grpc.py into proto_gen/ -// ────────────────────────────────────────────────────────────────────────── - -syntax = "proto3"; -package robonix.audio_driver; - -// ── Shared types ───────────────────────────────────────────────────────── -// Inlined for standalone build. Decoupled from speech_service.proto types. - -// Describes the audio format used by a stream. -message AudioConfig { - string encoding = 1; // PCM encoding name, e.g. "pcm_s16le", "pcm_s24le" - uint32 sample_rate_hz = 2; // sample rate in Hz (mic default: 16000, speaker default: 24000) - uint32 channels = 3; // 1 = mono, 2 = stereo - uint32 bits_per_sample = 4; // bits per sample (8, 16, 24, 32) -} - -// A single time-stamped audio chunk. Produced by mic capture, consumed by -// speaker playback. Sequence numbers are monotonic per stream for ordering. -message AudioChunk { - uint64 timestamp_ns = 1; // capture/playback timestamp in nanoseconds since epoch - bytes data = 2; // raw PCM bytes (no headers, no container) - uint32 sequence = 3; // monotonically increasing chunk counter - float duration_s = 4; // duration of this chunk in seconds (e.g. 0.1) -} - -// ── Mic interface (topic_out: driver produces, consumers subscribe) ────── -// -// Data flow: ALSA mic → arecord subprocess → MicDriver → gRPC stream → caller -// -// The mic driver captures at a fixed format (default 16kHz mono s16le). -// If audio_config is provided, it is logged but currently the driver captures -// at its configured rate (see environment overrides in node.py). - -message MicStreamRequest { - AudioConfig audio_config = 1; // optional: desired capture format. If empty, uses defaults. -} - -// ── Speaker interface (topic_in: consumers push, driver plays) ────────── -// -// Data flow: caller → gRPC client stream → SpeakerDriver → aplay subprocess → ALSA speaker -// -// The first message SHOULD include audio_config to set the playback format. -// Subsequent messages carry chunk.data for continuous playback. - -message SpeakerChunkRequest { - AudioChunk chunk = 1; // audio data to play - AudioConfig audio_config = 2; // optional: sent once in first message to configure playback -} - -message SpeakerStreamResponse { - bool ok = 1; // true if all chunks played successfully - string error = 2; // non-empty on failure -} - -// ── Services ──────────────────────────────────────────────────────────── -// Two separate services for mic and speaker, each registered with Atlas -// as a distinct interface under the robonix/prm/audio namespace. - -// Microphone capture service. -// Contract ID: robonix/prm/audio/mic -// Transport: gRPC server-streaming -// Mode: caller sends one request, server streams AudioChunk continuously -// until the caller disconnects or the mic stops. -service PrmAudioMic { - rpc Stream(MicStreamRequest) returns (stream AudioChunk); -} - -// Speaker playback service. -// Contract ID: robonix/prm/audio/speaker -// Transport: gRPC client-streaming -// Mode: caller streams audio chunks, driver plays them in real time. -// Returns a single response when the caller closes the stream. -service PrmAudioSpeaker { - rpc Stream(stream SpeakerChunkRequest) returns (SpeakerStreamResponse); -} diff --git a/rust/examples/packages/audio_driver/scripts/build.sh b/rust/examples/packages/audio_driver/scripts/build.sh index a98ca6f85..345e240e1 100644 --- a/rust/examples/packages/audio_driver/scripts/build.sh +++ b/rust/examples/packages/audio_driver/scripts/build.sh @@ -1,16 +1,10 @@ #!/usr/bin/env bash -# SPDX-License-Identifier: MulanPSL-2.0 set -euo pipefail PKG="${RBNX_PACKAGE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" -PROTO_SRC="$PKG/proto" -OUT_DIR="$PKG/proto_gen" +FLAGS=() +[[ "${RBNX_BUILD_CLEAN:-}" == "1" ]] && FLAGS+=(--clean) -mkdir -p "$OUT_DIR" - -python -m grpc_tools.protoc \ - -I"$PROTO_SRC" \ - --python_out="$OUT_DIR" \ - --grpc_python_out="$OUT_DIR" \ - "$PROTO_SRC/audio_driver.proto" +# rbnx codegen (generates proto stubs from ROS IDL + contracts) +rbnx codegen -p "$PKG" "${FLAGS[@]}" echo "[build] done." diff --git a/rust/examples/packages/speech_service/README.md b/rust/examples/packages/speech_service/README.md index 4bab7470e..76bdffcd1 100644 --- a/rust/examples/packages/speech_service/README.md +++ b/rust/examples/packages/speech_service/README.md @@ -1,14 +1,14 @@ # Speech Service -语音交互服务 — Robonix srv 层，提供 ASR（语音识别）、TTS（语音合成）、Dialog（语音对话）三类 gRPC 接口。 +Voice interaction service -- Robonix srv layer, providing three types of gRPC interfaces: ASR (Automatic Speech Recognition), TTS (Text-to-Speech), and Dialog (Voice Dialog). -## 架构定位 +## Architecture Position ``` ┌─────────────────────────────────────────────┐ │ Application Layer │ ├─────────────────────────────────────────────┤ -│ speech_service ← 你在这里 (robonix/srv/speech) │ +│ speech_service ← you are here (robonix/srv/speech) │ ├─────────────────────────────────────────────┤ │ audio_driver (robonix/prm/audio) │ ├─────────────────────────────────────────────┤ @@ -16,122 +16,122 @@ └─────────────────────────────────────────────┘ ``` -- **上层**：接收应用层文本请求（TTS）或返回识别结果（ASR） -- **下层**：从 audio_driver 接收原始 PCM 音频流 -- **自适应性**：自动将任意采样率/声道/编码转为 16kHz mono pcm_s16le，调用方无需预处理 +- **Upper layer**: Receives text requests from applications (TTS) or returns recognition results (ASR) +- **Lower layer**: Receives raw PCM audio streams from audio_driver +- **Adaptability**: Automatically converts any sample rate/channel/encoding to 16kHz mono pcm_s16le -- callers do not need to pre-process audio -## gRPC 接口 +## gRPC Interfaces -| 服务 | RPC | 模式 | 后端 | 说明 | -|------|-----|------|------|------| -| SpeechAsr | Recognize | Unary | Whisper (GPU FP16) | 一次性识别，适合完整语句 | -| SpeechAsr | RecognizeStream | Bidi-stream | FunASR Paraformer | 流式识别，600ms 粒度 | -| SpeechTts | Synthesize | Unary | Edge TTS | 一次性合成，返回完整 MP3 | -| SpeechTts | SynthesizeStream | Server-stream | Edge TTS | 流式合成，边生成边返回 | -| SpeechDialog | StartDialog | Server-stream | — | 语音对话会话管理 | +| Service | RPC | Mode | Backend | Description | +|---------|-----|------|---------|-------------| +| SpeechAsr | Recognize | Unary | Whisper (GPU FP16) | One-shot recognition, suitable for complete utterances | +| SpeechAsr | RecognizeStream | Bidi-stream | FunASR Paraformer | Streaming recognition, 600ms granularity | +| SpeechTts | Synthesize | Unary | Edge TTS | One-shot synthesis, returns complete MP3 | +| SpeechTts | SynthesizeStream | Server-stream | Edge TTS | Streaming synthesis, yields chunks as generated | +| SpeechDialog | StartDialog | Server-stream | — | Voice dialog session management | -## 目录结构 +## Directory Structure ``` speech_service/ ├── proto/ -│ └── speech_service.proto # 自包含 gRPC 定义（内联 AudioConfig/AudioChunk） -├── proto_gen/ # build 时生成的 *_pb2.py（git 忽略） +│ └── speech_service.proto # Self-contained gRPC definitions (inline AudioConfig/AudioChunk) +├── proto_gen/ # Generated *_pb2.py at build time (git-ignored) ├── scripts/ -│ └── build.sh # proto 代码生成 +│ └── build.sh # Proto code generation ├── speech_service/ -│ ├── __init__.py # 包入口 -│ ├── service.py # gRPC servicer + 后端引擎 + main() -│ └── audio_utils.py # 音频格式自适应（任意格式 → 16kHz mono s16le） -├── robonix_manifest.yaml # Robonix 包描述 +│ ├── __init__.py # Package entry point +│ ├── service.py # gRPC servicer + backend engines + main() +│ └── audio_utils.py # Audio format adaptation (any format → 16kHz mono s16le) +├── robonix_manifest.yaml # Robonix package descriptor ├── requirements.txt └── .gitignore ``` -## 快速启动 +## Quick Start -### 1. 安装依赖 +### 1. Install Dependencies ```bash pip install -r requirements.txt -# 核心依赖：grpcio, grpcio-tools, transformers, torch, funasr, edge-tts, scipy, numpy +# Core dependencies: grpcio, grpcio-tools, transformers, torch, funasr, edge-tts, scipy, numpy ``` -### 2. 生成 proto stubs +### 2. Generate Proto Stubs ```bash bash scripts/build.sh ``` -### 3. 启动服务 +### 3. Start the Service ```bash -# 正常模式（需要 GPU + 模型权重） +# Normal mode (requires GPU + model weights) export PYTHONPATH=$(pwd)/proto_gen:${PYTHONPATH:-} python -m speech_service.service -# CI 模式（不需要 GPU/模型，返回 mock 结果） +# CI mode (no GPU/model needed, returns mock results) SPEECH_CI_MODE=1 python -m speech_service.service -# 指定端口 +# Specify port SPEECH_PORT=50060 python -m speech_service.service -# 跳过 Atlas 注册 +# Skip Atlas registration SPEECH_STANDALONE=1 python -m speech_service.service ``` -### 4. 通过 Robonix 启动 +### 4. Launch via Robonix ```bash rbnx run com.robonix.example.speech_service ``` -## 环境变量 +## Environment Variables -| 变量 | 默认值 | 说明 | -|------|--------|------| -| `ASR_MODEL` | `whisper-large-merged` | Whisper 模型路径 | -| `ASR_DEVICE` | `cuda` | Torch 设备（cuda/cpu） | -| `ASR_CHUNK_LENGTH` | `30.0` | Whisper 长音频分片长度（秒） | -| `ASR_BATCH_SIZE` | `4` | Whisper 批处理大小 | -| `FUNASR_MODEL` | `paraformer-zh-streaming` | FunASR 流式模型 | -| `FUNASR_CHUNK_SIZE` | `[0,10,5]` | Paraformer chunk_size 参数 | -| `TTS_VOICE` | `zh-CN-XiaoxiaoNeural` | Edge TTS 声音名称 | -| `ROBONIX_ATLAS` | `localhost:50051` | Atlas 控制面地址 | -| `SPEECH_PORT` | `0`（自动分配） | gRPC 监听端口 | -| `SPEECH_BIND_ADDR` | `0.0.0.0` | gRPC 绑定地址 | -| `SPEECH_CI_MODE` | — | 设为 `1` 启用 mock 模式 | +| Variable | Default | Description | +|----------|---------|-------------| +| `ASR_MODEL` | `whisper-large-merged` | Whisper model path | +| `ASR_DEVICE` | `cuda` | Torch device (cuda/cpu) | +| `ASR_CHUNK_LENGTH` | `30.0` | Whisper long-audio chunk length (seconds) | +| `ASR_BATCH_SIZE` | `4` | Whisper batch size | +| `FUNASR_MODEL` | `paraformer-zh-streaming` | FunASR streaming model | +| `FUNASR_CHUNK_SIZE` | `[0,10,5]` | Paraformer chunk_size parameter | +| `TTS_VOICE` | `zh-CN-XiaoxiaoNeural` | Edge TTS voice name | +| `ROBONIX_ATLAS` | `localhost:50051` | Atlas control plane address | +| `SPEECH_PORT` | `0` (auto-assign) | gRPC listen port | +| `SPEECH_BIND_ADDR` | `0.0.0.0` | gRPC bind address | +| `SPEECH_CI_MODE` | — | Set to `1` to enable mock mode | -## 音频自适应 +## Audio Adaptation -`audio_utils.py` 处理管线： +`audio_utils.py` processing pipeline: ``` -输入音频（任意格式） +Input audio (any format) │ - ├─ 1. 解码 → float32 numpy [-1, 1] - │ 支持: pcm_s16le, pcm_f32le, pcm_s32le, pcm_u8, pcm_s24le, wav + ├─ 1. Decode → float32 numpy [-1, 1] + │ Supported: pcm_s16le, pcm_f32le, pcm_s32le, pcm_u8, pcm_s24le, wav │ - ├─ 2. 多声道混合 → 单声道 + ├─ 2. Multi-channel downmix → mono │ reshape(-1, channels).mean(axis=1) │ - ├─ 3. 重采样 → 16kHz - │ scipy.signal.resample_poly（多相 FIR 滤波器） + ├─ 3. Resample → 16kHz + │ scipy.signal.resample_poly (polyphase FIR filter) │ - └─ 4. 编码 → pcm_s16le + └─ 4. Encode → pcm_s16le ``` -调用方无需关心音频格式，服务端自动处理 8kHz/16kHz/48kHz/stereo 等输入。 +Callers do not need to worry about audio format -- the server automatically handles inputs such as 8kHz/16kHz/48kHz/stereo. -## Atlas 集成 +## Atlas Integration -启动时可选注册到 Atlas 控制面： +Optionally registers with the Atlas control plane at startup: -- **RegisterNode**: `com.robonix.services.speech`，namespace `robonix/srv/speech`，kind `service` +- **RegisterNode**: `com.robonix.services.speech`, namespace `robonix/srv/speech`, kind `service` - **DeclareInterface × 5**: asr, asr_stream, tts, tts_stream, dialog -Atlas 不可用时自动降级为独立运行模式。 +Automatically degrades to standalone mode when Atlas is unavailable. -## Proto 设计 +## Proto Design -`speech_service.proto` 是**自包含**的 — AudioConfig 和 AudioChunk 类型内联定义，不依赖外部 proto 文件。与 `audio_driver.proto` 中的同名类型解耦，各自独立演进。 +`speech_service.proto` is **self-contained** -- AudioConfig and AudioChunk types are defined inline, with no dependency on external proto files. They are decoupled from the identically-named types in `audio_driver.proto`, allowing each to evolve independently. diff --git a/rust/examples/packages/speech_service/proto/speech_service.proto b/rust/examples/packages/speech_service/docs/speech_service.proto.ref similarity index 100% rename from rust/examples/packages/speech_service/proto/speech_service.proto rename to rust/examples/packages/speech_service/docs/speech_service.proto.ref diff --git a/rust/examples/packages/speech_service/scripts/build.sh b/rust/examples/packages/speech_service/scripts/build.sh index bb743a4bb..5ab998d92 100755 --- a/rust/examples/packages/speech_service/scripts/build.sh +++ b/rust/examples/packages/speech_service/scripts/build.sh @@ -1,25 +1,21 @@ #!/usr/bin/env bash -# SPDX-License-Identifier: MulanPSL-2.0 set -euo pipefail PKG="${RBNX_PACKAGE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" -PROTO_SRC="$PKG/proto" -PROTO_DEPS="$PKG/../../../crates/robonix-interfaces/robonix_proto" -OUT_DIR="$PKG/proto_gen" - FLAGS=() [[ "${RBNX_BUILD_CLEAN:-}" == "1" ]] && FLAGS+=(--clean) -# Step 1: rbnx codegen (generates IDL-based protos: asr.proto, tts.proto, etc.) -rbnx codegen -p "$PKG" "${FLAGS[@]}" || true +# Step 1: rbnx codegen (generates proto stubs from ROS IDL + contracts) +rbnx codegen -p "$PKG" "${FLAGS[@]}" + +# Step 2: Download model weights (skip in CI mode or if SKIP_MODEL_DOWNLOAD=1) +if [[ "${SPEECH_CI_MODE:-}" != "1" && "${SKIP_MODEL_DOWNLOAD:-}" != "1" ]]; then + echo "[build] Downloading ASR model (openai/whisper-large-v3)..." + python3 -c "from transformers import pipeline; pipeline('automatic-speech-recognition', model='openai/whisper-large-v3', model_kwargs={'local_files_only': False})" || echo "[build] WARNING: Whisper model download failed. Service will fail to start ASR backend." -# Step 2: Generate gRPC service stubs from speech_service.proto -mkdir -p "$OUT_DIR" -python -m grpc_tools.protoc \ - -I"$OUT_DIR" \ - -I"$PROTO_DEPS" \ - -I"$PROTO_SRC" \ - --python_out="$OUT_DIR" \ - --grpc_python_out="$OUT_DIR" \ - "$PROTO_SRC/speech_service.proto" + echo "[build] Downloading FunASR model (paraformer-zh-streaming)..." + python3 -c "from funasr import AutoModel; AutoModel(model='paraformer-zh-streaming')" || echo "[build] WARNING: FunASR model download failed. Service will fail to start streaming ASR backend." +else + echo "[build] Skipping model download (CI mode or SKIP_MODEL_DOWNLOAD=1)." +fi echo "[build] done." diff --git a/rust/examples/packages/speech_service/speech_service/service.py b/rust/examples/packages/speech_service/speech_service/service.py index 5528b16d6..747f9f3bd 100644 --- a/rust/examples/packages/speech_service/speech_service/service.py +++ b/rust/examples/packages/speech_service/speech_service/service.py @@ -1,41 +1,45 @@ #!/usr/bin/env python3 -"""Speech service — the srv-layer voice interaction service for Robonix. +"""Speech service -- the srv-layer voice interaction service for Robonix. Architecture position: robonix/srv/speech - Sits ABOVE the prm layer (audio_driver) and BELOW the application layer - Receives raw audio from audio_driver via gRPC, returns transcriptions - Receives text from applications, returns synthesized audio (MP3) -Provides 3 gRPC services (5 RPCs total): +Provides 5 RPCs across 5 gRPC service contracts (from robonix_contracts.proto): - SpeechAsr (Automatic Speech Recognition): - - Recognize(req) → resp — one-shot, full-utterance transcription - - RecognizeStream(stream) → stream — real-time chunk-by-chunk transcription + SrvSpeechAsr (Automatic Speech Recognition -- one-shot): + Call(req) -> resp -- full-utterance transcription - SpeechTts (Text-to-Speech): - - Synthesize(req) → resp — one-shot, returns complete MP3 audio - - SynthesizeStream(req) → stream — streaming, yields MP3 chunks as generated + SrvSpeechAsrStream (Streaming ASR): + Stream(stream) -> stream -- real-time chunk-by-chunk transcription - SpeechDialog (Voice Dialog Session): - - StartDialog(req) → stream — managed session with state transitions + SrvSpeechTts (Text-to-Speech -- one-shot): + Call(req) -> resp -- returns complete MP3 audio + + SrvSpeechTtsStream (Streaming TTS): + Stream(req) -> stream -- yields MP3 chunks as generated + + SrvSpeechDialog (Voice Dialog Session): + Stream(req) -> stream -- managed session with state transitions Backend engines: - - Whisper (transformers pipeline) — one-shot ASR, GPU FP16, high accuracy - - FunASR Paraformer-zh-streaming — streaming ASR, 600ms granularity - - Edge TTS (Microsoft, free) — TTS synthesis, zh-CN-XiaoxiaoNeural + - Whisper (transformers pipeline) -- one-shot ASR, GPU FP16, high accuracy + - FunASR Paraformer-zh-streaming -- streaming ASR, 600ms granularity + - Edge TTS (Microsoft, free) -- TTS synthesis, zh-CN-XiaoxiaoNeural Audio adaptation: The service auto-adapts any input format (sample_rate, channels, encoding) to 16kHz mono pcm_s16le using speech_service.audio_utils.adapt_audio(). - Callers do NOT need to pre-process audio — the service handles it. + Callers do NOT need to pre-process audio -- the service handles it. Atlas integration: On startup, the service optionally registers with the Atlas control plane - (RegisterNode + DeclareInterface × 5). If Atlas is unavailable, the service + (RegisterNode + DeclareInterface x 5). If Atlas is unavailable, the service runs in standalone mode. Set SPEECH_STANDALONE=1 to skip registration. Environment variables: - ASR_MODEL Whisper model path (default: whisper-large-merged) + ASR_MODEL Whisper model path (default: openai/whisper-large-v3) ASR_DEVICE Torch device: cuda | cpu (default: cuda) ASR_CHUNK_LENGTH Whisper chunk length in seconds (default: 30.0) ASR_BATCH_SIZE Whisper batch size (default: 4) @@ -63,16 +67,16 @@ logging.basicConfig(level=logging.INFO, format="[speech-service] %(levelname)s %(message)s") log = logging.getLogger(__name__) -# ── Proto stub resolution ────────────────────────────────────────────────── +# -- Proto stub resolution --------------------------------------------------- # Walks up from this file's directory looking for proto_gen/ containing the -# generated *_pb2.py files. Adds it to sys.path so "import speech_service_pb2" -# works regardless of the working directory. +# codegen-generated *_pb2.py / *_pb2_grpc.py files. Adds it to sys.path so +# "import asr_pb2" etc. work regardless of the working directory. def _ensure_proto_gen() -> None: d = Path(__file__).resolve().parent while d.parent != d: pg = d / "proto_gen" - if pg.is_dir() and (pg / "speech_service_pb2.py").exists(): + if pg.is_dir() and (pg / "robonix_contracts_pb2_grpc.py").exists(): sys.path.insert(0, str(pg)) return d = d.parent @@ -80,30 +84,33 @@ def _ensure_proto_gen() -> None: _ensure_proto_gen() import grpc -import speech_service_pb2 as spb -import speech_service_pb2_grpc as spb_grpc +import asr_pb2 +import tts_pb2 +import speech_pb2 +import robonix_msg_pb2 +import robonix_contracts_pb2_grpc as contracts_grpc -# ── CI mock mode ────────────────────────────────────────────────────────── +# -- CI mock mode ------------------------------------------------------------ CI_MODE = os.environ.get("SPEECH_CI_MODE", "").strip() in ("1", "true", "yes") -# ── ASR Backend (Whisper) ───────────────────────────────────────────────── +# -- ASR Backend (Whisper) --------------------------------------------------- class WhisperASRBackend: """One-shot ASR backend using HuggingFace Whisper pipeline. - Loads a Whisper model (default: whisper-large-merged) onto GPU with FP16. - Best suited for complete utterances — not real-time streaming. + Loads a Whisper model (default: openai/whisper-large-v3) onto GPU with FP16. + Best suited for complete utterances -- not real-time streaming. Pipeline: - raw PCM bytes → numpy float32 → transformers pipeline → text - + automatic traditional→simplified Chinese conversion (OpenCC t2s) + raw PCM bytes -> numpy float32 -> transformers pipeline -> text + + automatic traditional-to-simplified Chinese conversion (OpenCC t2s) Config (environment): - ASR_MODEL — local model path or HuggingFace model ID - ASR_DEVICE — "cuda" or "cpu" - ASR_CHUNK_LENGTH — long-form audio chunk length in seconds - ASR_BATCH_SIZE — batch size for pipeline inference + ASR_MODEL -- local model path or HuggingFace model ID + ASR_DEVICE -- "cuda" or "cpu" + ASR_CHUNK_LENGTH -- long-form audio chunk length in seconds + ASR_BATCH_SIZE -- batch size for pipeline inference """ def __init__(self): @@ -121,6 +128,7 @@ def __init__(self): model=model_path, device=device, torch_dtype=torch.float16, + model_kwargs={"local_files_only": True}, ) self.chunk_length_s = chunk_length self.batch_size = batch_size @@ -183,17 +191,17 @@ def _t2s(text: str) -> str: class MockASRBackend: - """CI mock ASR — returns a fixed canned response, no model loaded. + """CI mock ASR -- returns a fixed canned response, no model loaded. Activated when SPEECH_CI_MODE=1. Useful for testing the gRPC layer without requiring GPU or model weights. """ def recognize(self, audio_bytes: bytes, encoding: str, sample_rate: int, language: str) -> dict: - return {"text": "[ci-mock] 你好世界", "confidence": 1.0} + return {"text": "[ci-mock] hello world", "confidence": 1.0} -# ── ASR Backend (FunASR Paraformer streaming) ───────────────────────────── +# -- ASR Backend (FunASR Paraformer streaming) -------------------------------- class FunASRStreamingBackend: """Streaming ASR backend using FunASR Paraformer-zh-streaming. @@ -203,17 +211,17 @@ class FunASRStreamingBackend: Model parameters: - Model: paraformer-zh-streaming (or FUNASR_MODEL env override) - - chunk_size: [0, 10, 5] — 0 left context, 10 current (600ms), 5 right (300ms) - - chunk_stride: 10 × 960 = 9600 samples = 600ms at 16kHz + - chunk_size: [0, 10, 5] -- 0 left context, 10 current (600ms), 5 right (300ms) + - chunk_stride: 10 x 960 = 9600 samples = 600ms at 16kHz - encoder_chunk_look_back: 4 (retain 4 past encoder chunks) - decoder_chunk_look_back: 1 (retain 1 past decoder chunk) The cache dict is passed across recognize_chunk() calls within a single - stream session. Each RecognizeStream gRPC call gets its own cache. + stream session. Each Stream gRPC call gets its own cache. Config (environment): - FUNASR_MODEL — model name or local path - FUNASR_CHUNK_SIZE — JSON array [left, current, right] + FUNASR_MODEL -- model name or local path + FUNASR_CHUNK_SIZE -- JSON array [left, current, right] """ def __init__(self): @@ -230,7 +238,7 @@ def __init__(self): log.info("Loading FunASR model %s (chunk_size=%s, stride=%d samples)...", model_name, self.chunk_size, self.chunk_stride) - self.model = AutoModel(model=model_name) + self.model = AutoModel(model=model_name, hub_kwargs={"local_files_only": True}) log.info("FunASR model loaded.") def recognize_chunk( @@ -245,11 +253,11 @@ def recognize_chunk( Args: audio_chunk: Raw PCM bytes (already adapted to 16kHz mono pcm_s16le). - Chunk size should be ~600ms (9600 samples × 2 bytes = 19200 bytes) + Chunk size should be ~600ms (9600 samples x 2 bytes = 19200 bytes) for optimal granularity, but any size works. cache: Mutable dict maintained across chunks in one session. The model stores encoder/decoder state here. Must be the same - dict for all chunks in one RecognizeStream call. + dict for all chunks in one Stream call. is_final: True on the last call to flush remaining state. Send empty audio_chunk with is_final=True to get the final result. encoding: Audio encoding (default pcm_s16le). @@ -292,17 +300,17 @@ def recognize_chunk( class MockASRStreamingBackend: - """CI mock streaming ASR — returns empty results during streaming, + """CI mock streaming ASR -- returns empty results during streaming, canned result on is_final. No model loaded. """ def recognize_chunk(self, audio_chunk, cache, is_final=False, encoding="pcm_s16le", sample_rate=16000): if is_final: - return [{"text": "[ci-mock-stream] 你好世界", "confidence": 1.0}] + return [{"text": "[ci-mock-stream] hello world", "confidence": 1.0}] return [{"text": "", "confidence": 0.0}] -# ── TTS Backend (Edge TTS) ─────────────────────────────────────────────── +# -- TTS Backend (Edge TTS) -------------------------------------------------- class EdgeTTSBackend: """TTS backend using Microsoft Edge TTS (free, no API key required). @@ -311,12 +319,12 @@ class EdgeTTSBackend: Services TTS endpoint. Outputs MP3 audio at 24kHz. Config (environment): - TTS_VOICE — voice name (default: zh-CN-XiaoxiaoNeural) + TTS_VOICE -- voice name (default: zh-CN-XiaoxiaoNeural) Other voices: zh-CN-YunxiNeural, en-US-JennyNeural, ja-JP-NanamiNeural Speed control: speed parameter is a multiplier. Internally converted to Edge TTS - rate string format: "+N%" or "-N%" where N = |speed-1| × 100. + rate string format: "+N%" or "-N%" where N = |speed-1| x 100. """ def __init__(self): @@ -349,7 +357,7 @@ async def synthesize(self, text: str, voice: str = "", speed: float = 1.0) -> by async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0): """Async generator that yields MP3 audio chunks as they're generated. - Useful for long text — the caller can start playback before the + Useful for long text -- the caller can start playback before the full audio is ready. Args: @@ -358,7 +366,7 @@ async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0 speed: Speed multiplier. Yields: - bytes — individual MP3 audio chunks from Edge TTS streaming. + bytes -- individual MP3 audio chunks from Edge TTS streaming. """ import edge_tts @@ -372,7 +380,7 @@ async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0 class MockTTSBackend: - """CI mock TTS — returns a minimal valid WAV file (silence). + """CI mock TTS -- returns a minimal valid WAV file (silence). Activated when SPEECH_CI_MODE=1. """ @@ -384,17 +392,17 @@ async def synthesize_stream(self, text: str, voice: str = "", speed: float = 1.0 yield data -# ── Dialog Session Manager ──────────────────────────────────────────────── +# -- Dialog Session Manager -------------------------------------------------- # Manages the lifecycle of voice dialog sessions. Each StartDialog call -# creates a session that tracks state (IDLE → LISTENING → PROCESSING → +# creates a session that tracks state (IDLE -> LISTENING -> PROCESSING -> # SPEAKING) until the client disconnects. class DialogSession: """Tracks state for a single voice dialog session. State machine: - IDLE → LISTENING → PROCESSING → SPEAKING → IDLE (loop) - Any state → ERROR (on failure) + IDLE -> LISTENING -> PROCESSING -> SPEAKING -> IDLE (loop) + Any state -> ERROR (on failure) Attributes: session_id: 8-char UUID prefix, unique per session. @@ -443,44 +451,50 @@ def remove_session(self, session_id: str) -> bool: return False -# ── gRPC Servicers ──────────────────────────────────────────────────────── +# -- gRPC Servicers ---------------------------------------------------------- # These classes implement the gRPC service interfaces defined in -# speech_service.proto. Each servicer wraps one or more backend engines. +# robonix_contracts.proto. Each servicer wraps one or more backend engines. +# Note: the codegen service methods are named Call / Stream (not Recognize +# or Synthesize) because the contract RPC is always called "Call" or "Stream". -class SpeechAsrServicer(spb_grpc.SpeechAsrServicer): - """ASR gRPC servicer — handles Recognize and RecognizeStream RPCs. +class SpeechAsrServicer(contracts_grpc.SrvSpeechAsrServicer): + """ASR gRPC servicer -- handles the Call RPC for one-shot speech recognition. - Delegates to: - - asr_backend (WhisperASRBackend) for one-shot Recognize - - stream_asr_backend (FunASRStreamingBackend) for streaming RecognizeStream + Delegates to WhisperASRBackend for transcription. - If a backend is None (failed to load at startup), the corresponding RPC - returns a clear error with actionable instructions instead of crashing. + If the backend is None (failed to load at startup), the RPC returns + UNAVAILABLE with actionable instructions instead of crashing. """ - def __init__(self, asr_backend, stream_asr_backend): + def __init__(self, asr_backend): self.asr_backend = asr_backend - self.stream_asr_backend = stream_asr_backend - def Recognize(self, request, context): - """Handle one-shot ASR: receive complete audio, return transcription.""" + def Call(self, request, context): + """Handle one-shot ASR: receive complete audio, return transcription. + + Request fields (from asr.proto Recognize_Request): + audio_data -- raw audio bytes + encoding -- audio encoding string (e.g. "pcm_s16le") + sample_rate_hz -- sample rate in Hz + language -- BCP-47 language tag + + Returns asr_pb2.Recognize_Response with text, confidence, error. + """ if self.asr_backend is None: context.set_code(grpc.StatusCode.UNAVAILABLE) context.set_details( "ASR backend not available. " - "Set ASR_MODEL to a local model path, or ensure network access " - "for auto-download from HuggingFace Hub. " + "Set ASR_MODEL to a local model path with pre-downloaded weights. " + "Runtime downloads are disabled (local_files_only=True). " "Current ASR_MODEL=" + os.environ.get("ASR_MODEL", "openai/whisper-large-v3") ) - return spb.RecognizeResponse() + return asr_pb2.Recognize_Response() from speech_service.audio_utils import adapt_audio encoding = request.encoding or "pcm_s16le" sample_rate = request.sample_rate_hz or 16000 - channels = request.channels or 1 - bits_per_sample = request.bits_per_sample or 16 language = request.language try: @@ -489,59 +503,76 @@ def Recognize(self, request, context): request.audio_data, encoding=encoding, sample_rate=sample_rate, - channels=channels, - bits_per_sample=bits_per_sample, ) result = self.asr_backend.recognize(audio_data, "pcm_s16le", 16000, language) - return spb.RecognizeResponse( + return asr_pb2.Recognize_Response( text=result["text"], confidence=result.get("confidence", 0.0), error="", ) except Exception as e: log.exception("ASR recognize failed") - return spb.RecognizeResponse(text="", confidence=0.0, error=str(e)) + return asr_pb2.Recognize_Response(text="", confidence=0.0, error=str(e)) + + +class SpeechAsrStreamServicer(contracts_grpc.SrvSpeechAsrStreamServicer): + """Streaming ASR gRPC servicer -- handles the Stream RPC for chunk-by-chunk + speech recognition. + + Delegates to FunASRStreamingBackend for real-time recognition. + + This is a bidirectional stream: the client sends AsrAudioChunk messages + (each containing an AudioChunk with raw audio data), and the server + yields RecognizeStreamEvent messages with partial/final transcriptions. + + Note: the stream carries no AudioConfig or language -- the service uses + defaults (16kHz mono pcm_s16le). Each chunk is adapted if needed. + """ + + def __init__(self, stream_asr_backend): + self.stream_asr_backend = stream_asr_backend - def RecognizeStream(self, request_iterator, context): - """Handle streaming ASR: receive audio chunks, yield partial/final results.""" + def Stream(self, request_iterator, context): + """Handle streaming ASR: receive audio chunks, yield partial/final results. + + Input: asr_pb2.AsrAudioChunk (has .chunk with .data field from robonix_msg.AudioChunk) + Output: asr_pb2.RecognizeStreamEvent + + Since the stream has no AudioConfig, we use defaults: 16kHz mono pcm_s16le. + """ if self.stream_asr_backend is None: context.set_code(grpc.StatusCode.UNAVAILABLE) context.set_details( "Streaming ASR backend not available. " - "Set FUNASR_MODEL to a local model path, or ensure network access " - "for auto-download. " + "Set FUNASR_MODEL to a local model path with pre-downloaded weights. " + "Runtime downloads are disabled (local_files_only=True). " "Current FUNASR_MODEL=" + os.environ.get("FUNASR_MODEL", "paraformer-zh-streaming") ) return - from speech_service.audio_utils import adapt_audio, parse_audio_config + from speech_service.audio_utils import adapt_audio cache = {} - language = "" - config = {"encoding": "pcm_s16le", "sample_rate": 16000, "channels": 1, "bits_per_sample": 16} chunk_count = 0 try: for req in request_iterator: - if req.audio_config and (req.audio_config.encoding or req.audio_config.sample_rate_hz): - config = parse_audio_config(req.audio_config) - if req.language: - language = req.language - + # Extract raw audio bytes from AsrAudioChunk.chunk.data chunk_data = bytes(req.chunk.data) if req.chunk and req.chunk.data else None if chunk_data is None: continue chunk_count += 1 - # Adapt each chunk to 16kHz mono pcm_s16le + # Adapt each chunk to 16kHz mono pcm_s16le using defaults + # (stream carries no AudioConfig, so assume pcm_s16le at 16kHz mono) adapted, _ = adapt_audio( chunk_data, - encoding=config["encoding"], - sample_rate=config["sample_rate"], - channels=config["channels"], - bits_per_sample=config["bits_per_sample"], + encoding="pcm_s16le", + sample_rate=16000, + channels=1, + bits_per_sample=16, ) results = self.stream_asr_backend.recognize_chunk( @@ -550,10 +581,10 @@ def RecognizeStream(self, request_iterator, context): ) for r in results: if r.get("text"): - yield spb.RecognizeStreamResponse( + yield asr_pb2.RecognizeStreamEvent( event_type=0, text=r["text"], confidence=r.get("confidence", 0.0), - language=language, + language="", ) # Final flush @@ -564,36 +595,44 @@ def RecognizeStream(self, request_iterator, context): ) for r in final_results: if r.get("text"): - yield spb.RecognizeStreamResponse( + yield asr_pb2.RecognizeStreamEvent( event_type=1, text=r["text"], confidence=r.get("confidence", 0.0), - language=language, is_final=True, + language="", is_final=True, ) else: - yield spb.RecognizeStreamResponse( + yield asr_pb2.RecognizeStreamEvent( event_type=2, error="No audio data received", ) except Exception as e: log.exception("ASR stream recognize failed") - yield spb.RecognizeStreamResponse(event_type=2, error=str(e)) + yield asr_pb2.RecognizeStreamEvent(event_type=2, error=str(e)) -class SpeechTtsServicer(spb_grpc.SpeechTtsServicer): - """TTS gRPC servicer — handles Synthesize and SynthesizeStream RPCs. +class SpeechTtsServicer(contracts_grpc.SrvSpeechTtsServicer): + """TTS gRPC servicer -- handles the Call RPC for one-shot text-to-speech. Delegates to EdgeTTSBackend for audio generation. Output format: MP3 at 24kHz (fixed by Edge TTS). Note: Edge TTS uses asyncio internally, but gRPC servicers are sync. - Synthesize uses asyncio.run(), SynthesizeStream creates a dedicated - event loop to avoid "cannot run from running loop" conflicts. + Call uses asyncio.run(). """ def __init__(self, tts_backend): self.tts_backend = tts_backend - def Synthesize(self, request, context): - """Handle one-shot TTS: receive text, return complete MP3 audio.""" + def Call(self, request, context): + """Handle one-shot TTS: receive text, return complete MP3 audio. + + Request fields (from tts.proto Synthesize_Request): + text -- text to synthesize + language -- BCP-47 language tag + voice -- voice name override + speed -- speed multiplier (1.0 = normal) + + Returns tts_pb2.Synthesize_Response with audio_data, encoding, sample_rate_hz. + """ if self.tts_backend is None: context.set_code(grpc.StatusCode.UNAVAILABLE) context.set_details( @@ -601,7 +640,7 @@ def Synthesize(self, request, context): "Edge TTS requires network access to Microsoft Cognitive Services. " "Check your network connection or set SPEECH_CI_MODE=1 for testing." ) - return spb.SynthesizeResponse() + return tts_pb2.Synthesize_Response() text = request.text voice = request.voice @@ -609,7 +648,7 @@ def Synthesize(self, request, context): try: audio_data = asyncio.run(self.tts_backend.synthesize(text, voice, speed)) - return spb.SynthesizeResponse( + return tts_pb2.Synthesize_Response( audio_data=audio_data, encoding="mp3", sample_rate_hz=24000, @@ -617,10 +656,38 @@ def Synthesize(self, request, context): ) except Exception as e: log.exception("TTS synthesize failed") - return spb.SynthesizeResponse(audio_data=b"", error=str(e)) + return tts_pb2.Synthesize_Response(audio_data=b"", error=str(e)) - def SynthesizeStream(self, request, context): - """Handle streaming TTS: receive text, yield MP3 audio chunks.""" + +class SpeechTtsStreamServicer(contracts_grpc.SrvSpeechTtsStreamServicer): + """Streaming TTS gRPC servicer -- handles the Stream RPC for chunk-by-chunk + text-to-speech synthesis. + + Delegates to EdgeTTSBackend for audio generation. + Output format: MP3 at 24kHz (fixed by Edge TTS). + + This is a server stream: unary request with text/voice/speed, + streams back tts_pb2.SynthesizeAudioChunk messages. + + Note: Edge TTS uses asyncio internally. A dedicated event loop is + created per call to avoid "cannot run from running loop" conflicts. + """ + + def __init__(self, tts_backend): + self.tts_backend = tts_backend + + def Stream(self, request, context): + """Handle streaming TTS: receive text, yield MP3 audio chunks. + + Request fields (from tts.proto SynthesizeStream_Request): + text -- text to synthesize + language -- BCP-47 language tag + voice -- voice name override + speed -- speed multiplier + audio_config -- AudioConfig (reserved, not used by Edge TTS) + + Yields tts_pb2.SynthesizeAudioChunk (wrapping robonix_msg.AudioChunk). + """ if self.tts_backend is None: context.set_code(grpc.StatusCode.UNAVAILABLE) context.set_details( @@ -636,19 +703,26 @@ def SynthesizeStream(self, request, context): try: loop = asyncio.new_event_loop() async_gen = self.tts_backend.synthesize_stream(text, voice, speed) + seq = 0 while True: try: chunk_data = loop.run_until_complete(async_gen.__anext__()) - yield spb.SynthesizeStreamResponse( - audio_data=chunk_data, + yield tts_pb2.SynthesizeAudioChunk( + chunk=robonix_msg_pb2.AudioChunk( + data=chunk_data, + sequence=seq, + ), encoding="mp3", sample_rate_hz=24000, is_final=False, ) + seq += 1 except StopAsyncIteration: - yield spb.SynthesizeStreamResponse( - audio_data=b"", encoding="mp3", - sample_rate_hz=24000, is_final=True, + yield tts_pb2.SynthesizeAudioChunk( + chunk=robonix_msg_pb2.AudioChunk(data=b""), + encoding="mp3", + sample_rate_hz=24000, + is_final=True, ) break loop.close() @@ -658,10 +732,10 @@ def SynthesizeStream(self, request, context): context.set_details(str(e)) -class SpeechDialogServicer(spb_grpc.SpeechDialogServicer): - """Dialog gRPC servicer — handles StartDialog RPC. +class SpeechDialogServicer(contracts_grpc.SrvSpeechDialogServicer): + """Dialog gRPC servicer -- handles the Stream RPC for voice dialog sessions. - Creates a DialogSession and streams state updates to the client. + Creates a DialogSession and streams DialogEvent updates to the client. The session stays alive until the client disconnects, then is automatically cleaned up. @@ -672,16 +746,31 @@ class SpeechDialogServicer(spb_grpc.SpeechDialogServicer): def __init__(self, dialog_manager): self.dialog_manager = dialog_manager - def StartDialog(self, request, context): + def Stream(self, request, context): + """Handle dialog session: create session, stream state updates. + + Request fields (from speech.proto StartDialog_Request): + language -- BCP-47 language tag + enable_vad -- enable voice activity detection + audio_config -- AudioConfig for the session + + Yields speech_pb2.StartDialog_Response wrapping robonix_msg.DialogEvent. + """ language = request.language or "zh-CN" enable_vad = request.enable_vad if hasattr(request, "enable_vad") else True session = self.dialog_manager.create_session(language, enable_vad) - yield spb.StartDialogResponse( - session_id=session.session_id, - state=DialogSession.STATE_MAP["IDLE"], - error="", + yield speech_pb2.StartDialog_Response( + event=robonix_msg_pb2.DialogEvent( + session_id=session.session_id, + event_type=DialogSession.STATE_MAP["IDLE"], + text="", + confidence=0.0, + language=session.language, + is_final=False, + error="", + ) ) try: @@ -693,7 +782,7 @@ def StartDialog(self, request, context): self.dialog_manager.remove_session(session.session_id) -# ── Atlas registration (optional) ───────────────────────────────────────── +# -- Atlas registration (optional) ------------------------------------------- # Registers the speech service with the Robonix Atlas control plane so that # other nodes can discover it. If Atlas is unavailable or the runtime protos # are not installed, the service runs in standalone mode. @@ -704,13 +793,13 @@ def _register_with_atlas(port: int) -> None: Registration flow: 1. RegisterNode: node_id="com.robonix.services.speech", namespace="robonix/srv/speech", kind="service" - 2. DeclareInterface × 5: one per RPC (asr, asr_stream, tts, + 2. DeclareInterface x 5: one per contract (asr, asr_stream, tts, tts_stream, dialog), each with its gRPC mode. 3. Atlas returns allocated_endpoint for each interface. If robonix_runtime_pb2 is not importable, or Atlas is unreachable, logs a warning and continues. The service is fully functional without - Atlas — it just won't be discoverable by other nodes. + Atlas -- it just won't be discoverable by other nodes. Args: port: The gRPC port the service is listening on. @@ -759,7 +848,7 @@ def _register_with_atlas(port: int) -> None: log.warning("Atlas not available (%s), running standalone", e.code()) -# ── Main ────────────────────────────────────────────────────────────────── +# -- Main -------------------------------------------------------------------- # Entry point. Called via `python -m speech_service.service` or via the # robonix_manifest.yaml start command: `exec python -m speech_service.service`. @@ -772,14 +861,14 @@ def main() -> None: - Failed backends are set to None; servicers return UNAVAILABLE - If ALL backends fail, exit with error 2. Create gRPC server with thread pool (8 workers) - 3. Register all 3 servicers (ASR, TTS, Dialog) + 3. Register all 5 servicers (ASR, ASR Stream, TTS, TTS Stream, Dialog) 4. Bind to SPEECH_BIND_ADDR:SPEECH_PORT (default 0.0.0.0:0 = auto) 5. Optionally register with Atlas control plane 6. Block until server terminates """ log.info("Starting speech service (ci_mode=%s)", CI_MODE) - # ── Initialize backends ────────────────────────────────────────────── + # -- Initialize backends ------------------------------------------------- asr_backend = None asr_stream_backend = None tts_backend = None @@ -793,65 +882,65 @@ def main() -> None: try: asr_backend = WhisperASRBackend() except Exception as e: - log.error("╔══════════════════════════════════════════════════════════════╗") - log.error("║ Whisper ASR backend FAILED to initialize ║") - log.error("║ Error: %s", str(e)[:51]) - log.error("║ ║") - log.error("║ To fix, choose ONE of: ║") - log.error("║ 1. Ensure network access → auto-download from HuggingFace║") - log.error("║ 2. Set ASR_MODEL=/path/to/local/whisper/model ║") - log.error("║ 3. Set SPEECH_CI_MODE=1 for testing (mock backend) ║") - log.error("║ ║") - log.error("║ The service will start without ASR (Recognize → UNAVAILABLE║") - log.error("╚══════════════════════════════════════════════════════════════╝") + log.error("+============================================================+") + log.error("| Whisper ASR backend FAILED to initialize |") + log.error("| Error: %s", str(e)[:51]) + log.error("| |") + log.error("| To fix, choose ONE of: |") + log.error("| 1. Pre-download model and set ASR_MODEL=/path/to/model |") + log.error("| 2. Set SPEECH_CI_MODE=1 for testing (mock backend) |") + log.error("| |") + log.error("| Note: local_files_only=True -- no runtime downloads |") + log.error("| The service will start without ASR (Call -> UNAVAILABLE) |") + log.error("+============================================================+") asr_backend = None # FunASR Paraformer (streaming ASR) try: asr_stream_backend = FunASRStreamingBackend() except Exception as e: - log.error("╔══════════════════════════════════════════════════════════════╗") - log.error("║ FunASR streaming ASR backend FAILED to initialize ║") - log.error("║ Error: %s", str(e)[:51]) - log.error("║ ║") - log.error("║ To fix, choose ONE of: ║") - log.error("║ 1. pip install funasr + network access ║") - log.error("║ 2. Set FUNASR_MODEL=/path/to/local/paraformer/model ║") - log.error("║ 3. Set SPEECH_CI_MODE=1 for testing ║") - log.error("║ ║") - log.error("║ Streaming ASR (RecognizeStream) will be UNAVAILABLE ║") - log.error("╚══════════════════════════════════════════════════════════════╝") + log.error("+============================================================+") + log.error("| FunASR streaming ASR backend FAILED to initialize |") + log.error("| Error: %s", str(e)[:51]) + log.error("| |") + log.error("| To fix, choose ONE of: |") + log.error("| 1. Pre-download model and set FUNASR_MODEL=/path/to/model|") + log.error("| 2. Set SPEECH_CI_MODE=1 for testing |") + log.error("| |") + log.error("| Note: local_files_only=True -- no runtime downloads |") + log.error("| Streaming ASR (Stream) will be UNAVAILABLE |") + log.error("+============================================================+") asr_stream_backend = None # Edge TTS try: tts_backend = EdgeTTSBackend() except Exception as e: - log.error("╔══════════════════════════════════════════════════════════════╗") - log.error("║ Edge TTS backend FAILED to initialize ║") - log.error("║ Error: %s", str(e)[:51]) - log.error("║ ║") - log.error("║ To fix: ensure network access to Microsoft TTS endpoint ║") - log.error("║ or set SPEECH_CI_MODE=1 for testing ║") - log.error("║ ║") - log.error("║ TTS (Synthesize/SynthesizeStream) will be UNAVAILABLE ║") - log.error("╚══════════════════════════════════════════════════════════════╝") + log.error("+============================================================+") + log.error("| Edge TTS backend FAILED to initialize |") + log.error("| Error: %s", str(e)[:51]) + log.error("| |") + log.error("| To fix: ensure network access to Microsoft TTS endpoint |") + log.error("| or set SPEECH_CI_MODE=1 for testing |") + log.error("| |") + log.error("| TTS (Call/Stream) will be UNAVAILABLE |") + log.error("+============================================================+") tts_backend = None # Bail out if nothing is available available = [b is not None for b in (asr_backend, asr_stream_backend, tts_backend)] if not any(available): - log.error("All backends failed — cannot start service.") + log.error("All backends failed -- cannot start service.") log.error("Set SPEECH_CI_MODE=1 for testing, or fix model/network issues above.") sys.exit(1) - # ── Startup summary ────────────────────────────────────────────────── + # -- Startup summary ----------------------------------------------------- log.info("Backend status:") - log.info(" Whisper ASR (Recognize): %s", + log.info(" Whisper ASR (Call): %s", "OK" if asr_backend is not None else "UNAVAILABLE") - log.info(" FunASR (RecognizeStream): %s", + log.info(" FunASR (Stream): %s", "OK" if asr_stream_backend is not None else "UNAVAILABLE") - log.info(" Edge TTS (Synthesize): %s", + log.info(" Edge TTS (Call/Stream): %s", "OK" if tts_backend is not None else "UNAVAILABLE") dialog_manager = DialogManager() @@ -859,13 +948,13 @@ def main() -> None: # Start gRPC server server = grpc.server(futures.ThreadPoolExecutor(max_workers=8)) - # Register servicers - spb_grpc.add_SpeechAsrServicer_to_server( - SpeechAsrServicer(asr_backend, asr_stream_backend), server) - spb_grpc.add_SpeechTtsServicer_to_server( - SpeechTtsServicer(tts_backend), server) - spb_grpc.add_SpeechDialogServicer_to_server( - SpeechDialogServicer(dialog_manager), server) + # Register servicers (5 separate contracts from robonix_contracts.proto) + spb_grpc = contracts_grpc # for add_*_to_server calls + spb_grpc.add_SrvSpeechAsrServicer_to_server(SpeechAsrServicer(asr_backend), server) + spb_grpc.add_SrvSpeechAsrStreamServicer_to_server(SpeechAsrStreamServicer(asr_stream_backend), server) + spb_grpc.add_SrvSpeechTtsServicer_to_server(SpeechTtsServicer(tts_backend), server) + spb_grpc.add_SrvSpeechTtsStreamServicer_to_server(SpeechTtsStreamServicer(tts_backend), server) + spb_grpc.add_SrvSpeechDialogServicer_to_server(SpeechDialogServicer(dialog_manager), server) # Bind port bind_addr = os.environ.get("SPEECH_BIND_ADDR", "0.0.0.0") From 9a1b0c05f416c0eab7a8ba195b0734046ecb2822 Mon Sep 17 00:00:00 2001 From: HustWolfzzb <1184827350@qq.com> Date: Fri, 24 Apr 2026 10:01:38 +0000 Subject: [PATCH 7/7] fix(build): register asr/tts/speech proto modules for tonic Add asr, tts, speech modules to contract_proto_modules.rs so tonic::include_proto! resolves the new message types referenced by robonix_contracts.proto services. --- .../robonix_proto/contract_proto_modules.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rust/crates/robonix-interfaces/robonix_proto/contract_proto_modules.rs b/rust/crates/robonix-interfaces/robonix_proto/contract_proto_modules.rs index 37993d9fa..fa7b8fd43 100644 --- a/rust/crates/robonix-interfaces/robonix_proto/contract_proto_modules.rs +++ b/rust/crates/robonix-interfaces/robonix_proto/contract_proto_modules.rs @@ -56,3 +56,15 @@ pub mod contracts { tonic::include_proto!("robonix.contracts"); } +pub mod asr { + tonic::include_proto!("robonix.asr"); +} + +pub mod tts { + tonic::include_proto!("robonix.tts"); +} + +pub mod speech { + tonic::include_proto!("robonix.speech"); +} +