diff --git a/config.example.yaml b/config.example.yaml index fe30e62..2c3b5a9 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -56,6 +56,10 @@ wake_word: vad_threshold: 0.5 # silero-vad probability threshold vad_gate: 0.0 # >0: wake only when openWakeWord's own VAD agrees (fewer false wakes from TV/music) silence_ms: 800 # end-of-utterance silence + voice_gate_ratio: 3.0 # how much louder than the background a frame must be to count as + # the user talking — lets recording end while your own speakers play + # music (Silero rates music as speech). 0 = off, raise if commands + # get cut off, lower if music keeps the recorder running. mcp: servers_config: "mcp_servers.json" diff --git a/docs/AUDIT.md b/docs/AUDIT.md index edc05a7..562b907 100644 --- a/docs/AUDIT.md +++ b/docs/AUDIT.md @@ -609,4 +609,5 @@ stt: hotwords: "Javis, Esszimmer, Homey" # bias Whisper towards these words wake_word: vad_gate: 0.5 # wake only when openWakeWord's VAD agrees + voice_gate_ratio: 3.0 # end recording when the *user* stops, not the music ``` diff --git a/javis-slave/README.md b/javis-slave/README.md index 7763b2a..b02e5a0 100644 --- a/javis-slave/README.md +++ b/javis-slave/README.md @@ -27,6 +27,11 @@ screen (GUI mode) and played back through the device's speakers. 4. **Turns** — After the wake word, the utterance is recorded locally (Silero VAD or RMS end-pointing), uploaded as WAV to `POST /api/v1/turn`, and the master's reply text + synthesized speech are returned and played. + Recording ends when *your voice* stops: Silero rates music and TV as + speech, so a loudness gate (`wake_word.voice_gate_ratio`, pushed by the + master) only counts audio that is clearly louder than the background the + wake-word loop measured. Without it, music from this device's own speakers + kept the recorder running until the hard length cap. When the master marks a reply as expecting a follow-up, the slave starts listening immediately after playback without requiring another wake word. If the master's turn fails it returns its spoken error phrase instead, so a diff --git a/javis-slave/src/javis_slave/api_client.py b/javis-slave/src/javis_slave/api_client.py index 89575c7..a8a7e83 100644 --- a/javis-slave/src/javis_slave/api_client.py +++ b/javis-slave/src/javis_slave/api_client.py @@ -58,6 +58,8 @@ def _wake_word_from(payload: dict) -> WakeWordSettings: threshold=float(ww.get("threshold", 0.5)), vad_threshold=float(ww.get("vad_threshold", 0.5)), silence_ms=int(ww.get("silence_ms", 800)), + # Older masters do not send this; keep the satellite's own default. + voice_gate_ratio=float(ww.get("voice_gate_ratio", 3.0)), ) diff --git a/javis-slave/src/javis_slave/audio_engine.py b/javis-slave/src/javis_slave/audio_engine.py index fdeb5ba..d4ccb28 100644 --- a/javis-slave/src/javis_slave/audio_engine.py +++ b/javis-slave/src/javis_slave/audio_engine.py @@ -104,7 +104,10 @@ class AudioEngine: ) try: await self.mic.start() - self.vad = make_vad(threshold=self.cfg.wake_word.vad_threshold) + self.vad = make_vad( + threshold=self.cfg.wake_word.vad_threshold, + voice_gate_ratio=self.cfg.wake_word.voice_gate_ratio, + ) self.wake = await self._load_wake(self.cfg.wake_word.model, self.cfg.wake_word.threshold) except BaseException: # The caller retries with a fresh engine; an open stream left behind @@ -261,10 +264,14 @@ class AudioEngine: self.on_state("muted") return False score = self.wake.feed(chunk) + # Everything heard before the wake word is background (music, TV, + # this device's own speakers), so the recorder's voice gate is + # already calibrated when the turn starts. + rms = self.vad.observe(chunk) if hasattr(self.vad, "observe") else ( + float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0) now = time.monotonic() if now - last_pub > 0.1: last_pub = now - rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0 self.last_rms = rms self.last_score = float(score) self.on_mic_level(rms, float(score)) @@ -386,8 +393,8 @@ class AudioEngine: self.on_room(room) changed = True old = self.cfg.wake_word - if (ww.model, ww.threshold, ww.vad_threshold, ww.silence_ms) != ( - old.model, old.threshold, old.vad_threshold, old.silence_ms): + if (ww.model, ww.threshold, ww.vad_threshold, ww.silence_ms, ww.voice_gate_ratio) != ( + old.model, old.threshold, old.vad_threshold, old.silence_ms, old.voice_gate_ratio): log.info(f"wake-word settings updated by master: {ww}") rebuild_model = ww.model != old.model self.cfg.wake_word = ww @@ -397,7 +404,8 @@ class AudioEngine: spawn(self._swap_wake_model(ww.model, ww.threshold)) else: self.wake.threshold = ww.threshold - self.vad = make_vad(threshold=ww.vad_threshold) + self.vad = make_vad( + threshold=ww.vad_threshold, voice_gate_ratio=ww.voice_gate_ratio) changed = True return changed diff --git a/javis-slave/src/javis_slave/audio_io/capture.py b/javis-slave/src/javis_slave/audio_io/capture.py index 140567c..b74d221 100644 --- a/javis-slave/src/javis_slave/audio_io/capture.py +++ b/javis-slave/src/javis_slave/audio_io/capture.py @@ -57,11 +57,13 @@ async def record_utterance( silent_run = 0 seen_speech = False chunks_seen = 0 + background = 0.0 async for chunk in mic.chunks(stale_timeout=stale_timeout): chunks_seen += 1 collected.append(chunk) result = vad.process(chunk) + background = result.background if result.is_speech: seen_speech = True @@ -70,9 +72,10 @@ async def record_utterance( silent_run += 1 if seen_speech and silent_run >= silence_chunks_needed: + log.debug(f"speech ended (background rms={background:.4f})") break if not seen_speech and chunks_seen >= grace_chunks: - log.debug("no speech during grace window") + log.debug(f"no speech during grace window (background rms={background:.4f})") return None if chunks_seen >= max_chunks: log.info("max utterance length reached") diff --git a/javis-slave/src/javis_slave/audio_io/vad.py b/javis-slave/src/javis_slave/audio_io/vad.py index 1f6cdaf..69c5318 100644 --- a/javis-slave/src/javis_slave/audio_io/vad.py +++ b/javis-slave/src/javis_slave/audio_io/vad.py @@ -5,8 +5,9 @@ diagnostics helper so this module has no dependency on the ``javis`` package. """ from __future__ import annotations +from collections import deque from dataclasses import dataclass -from typing import Protocol +from typing import Deque, Protocol import numpy as np @@ -19,6 +20,10 @@ log = get_logger("audio.vad") class VADResult: is_speech: bool probability: float + # Loudness of this frame and the background level it was judged against. + # Both are 0.0 for VADs that do not track loudness. + rms: float = 0.0 + background: float = 0.0 class VAD(Protocol): @@ -49,16 +54,129 @@ class NullVAD: if chunk.size == 0: return VADResult(False, 0.0) rms = float(np.sqrt(np.mean(np.square(chunk.astype(np.float32))))) - return VADResult(is_speech=rms >= self.threshold, probability=rms) + return VADResult(is_speech=rms >= self.threshold, probability=rms, rms=rms) -def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD: - """Construct a VAD, falling back to NullVAD if torch/silero fails to load.""" +# A frame must be at least this loud (RMS, normalized float32) to count as +# someone talking. Without the floor the gate would collapse to ~0 in a silent +# room and every faint noise would read as speech. +_MIN_VOICE_RMS = 0.006 +# Speech has short gaps, so a low percentile of the recent window still tracks +# the room even while the gate is open. +_BACKGROUND_PERCENTILE = 20.0 +# Once the user is already talking the gate drops to this fraction of the onset +# ratio, so a trailing quiet syllable is not clipped off mid-word. +_RELEASE_SCALE = 0.55 + + +def frame_rms(chunk: np.ndarray) -> float: + """RMS loudness of one mono float32 frame (0.0 for an empty frame).""" + if chunk is None or len(chunk) == 0: + return 0.0 + return float(np.sqrt(np.mean(np.square(np.asarray(chunk, dtype=np.float32))))) + + +class SpeechEndpointer: + """Reports only the *near-field* talker as speech. + + Silero answers "is this audio voice-like?", which music, TV and podcasts + satisfy just as well as the user does. When the satellite plays music + through its own speakers the VAD therefore never goes quiet, so a recording + never detects the end of the command and runs until the hard length cap. + + This wrapper adds the missing distinction: it tracks how loud the + surrounding audio is and only reports speech while a frame is clearly + louder than that background — which is exactly what separates someone + talking into the device from steady playback. Idle audio fed through + :meth:`observe` (the wake-word loop does) means the background is already + known by the time a recording starts. + + Implements the :class:`VAD` protocol, so it can be used anywhere a VAD is + expected. With ``onset_ratio <= 1.0`` the gate is disabled and the inner + VAD's verdict is passed through unchanged. + """ + + def __init__(self, vad: VAD, *, onset_ratio: float = 3.0, + window_s: float = 1.5, chunk_ms: int = 32) -> None: + self.vad = vad + self.sample_rate = vad.sample_rate + self.onset_ratio = max(0.0, float(onset_ratio)) + self.release_ratio = max(1.0, self.onset_ratio * _RELEASE_SCALE) + window = max(4, int(window_s * 1000.0 / max(1, chunk_ms))) + self._levels: Deque[float] = deque(maxlen=window) + self._in_speech = False + + @property + def threshold(self) -> float: + return self.vad.threshold + + @threshold.setter + def threshold(self, value: float) -> None: + self.vad.threshold = value + + @property + def enabled(self) -> bool: + """False when the loudness gate is off (plain VAD behaviour).""" + return self.onset_ratio > 1.0 + + @property + def background(self) -> float: + """Background loudness of the recent window; 0.0 until audio is seen.""" + if not self._levels: + return 0.0 + levels = sorted(self._levels) + idx = min(len(levels) - 1, + int(len(levels) * _BACKGROUND_PERCENTILE / 100.0)) + return levels[idx] + + def observe(self, chunk: np.ndarray) -> float: + """Feed audio heard while *not* recording; returns the frame's RMS. + + Anything heard outside a turn is background by definition, so the gate + is calibrated before the user even says the wake word. + """ + rms = frame_rms(chunk) + self._levels.append(rms) + self._in_speech = False + return rms + + def reset(self) -> None: + """Start a new utterance. Deliberately keeps the learned background.""" + self._in_speech = False + self.vad.reset() + + def process(self, chunk: np.ndarray) -> VADResult: + rms = frame_rms(chunk) + background = self.background + inner = self.vad.process(chunk) + if not self.enabled: + self._levels.append(rms) + return VADResult(inner.is_speech, inner.probability, rms, background) + ratio = self.release_ratio if self._in_speech else self.onset_ratio + gate = max(_MIN_VOICE_RMS, background * ratio) + is_voice = bool(inner.is_speech) and rms >= gate + self._in_speech = is_voice + if not is_voice: + # Only non-voice frames describe the background; letting speech in + # would let a long sentence raise the gate onto itself and cut the + # user off mid-utterance. + self._levels.append(rms) + return VADResult(is_voice, inner.probability, rms, background) + + +def make_vad(sample_rate: int = 16000, threshold: float = 0.5, + voice_gate_ratio: float = 3.0, chunk_ms: int = 32) -> VAD: + """Construct the capture VAD, falling back to NullVAD if torch/silero fails. + + The VAD is wrapped in a :class:`SpeechEndpointer` so that playback from the + satellite's own speakers is not mistaken for the user talking. + """ try: - return SileroVAD(sample_rate=sample_rate, threshold=threshold) + base: VAD = SileroVAD(sample_rate=sample_rate, threshold=threshold) except Exception as e: # noqa: BLE001 log.error(f"SileroVAD unavailable: {e}") - return NullVAD(sample_rate=sample_rate, threshold=threshold) + base = NullVAD(sample_rate=sample_rate, threshold=threshold) + return SpeechEndpointer(base, onset_ratio=voice_gate_ratio, chunk_ms=chunk_ms) class SileroVAD: diff --git a/javis-slave/src/javis_slave/config.py b/javis-slave/src/javis_slave/config.py index bfd95fb..d975583 100644 --- a/javis-slave/src/javis_slave/config.py +++ b/javis-slave/src/javis_slave/config.py @@ -40,6 +40,10 @@ class WakeWordSettings: threshold: float = 0.5 vad_threshold: float = 0.5 silence_ms: int = 800 + # How much louder than the background a frame must be to count as the user + # talking, so recording ends when the user stops even while this device's + # own speakers keep playing. 0 = off. + voice_gate_ratio: float = 3.0 @dataclass diff --git a/src/javis/agent/orchestrator.py b/src/javis/agent/orchestrator.py index f7a3ada..2b22d20 100644 --- a/src/javis/agent/orchestrator.py +++ b/src/javis/agent/orchestrator.py @@ -362,7 +362,10 @@ class Orchestrator: self.state: AgentState = AgentState.IDLE self.mic: MicStream | None = MicStream(config) if config.stt.enabled else None - self.vad = make_vad(threshold=config.wake_word.vad_threshold) + self.vad = make_vad( + threshold=config.wake_word.vad_threshold, + voice_gate_ratio=config.wake_word.voice_gate_ratio, + ) # Voice input requires STT; if STT is disabled, also disable wake word. self.stt = Transcriber(config) if config.stt.enabled else None self.wake = None @@ -952,10 +955,15 @@ class Orchestrator: last_pub = 0.0 async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S): score = self.wake.feed(chunk) + # Everything heard before the wake word is background (music, TV, + # this device's own speakers), so the recorder's voice gate is + # already calibrated when the turn starts. + observe = getattr(self.vad, "observe", None) + rms = observe(chunk) if observe is not None else ( + float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0) now = time.monotonic() if now - last_pub > 0.1: # ~10 Hz UI updates last_pub = now - rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0 await self.bus.publish("mic_level", {"rms": rms, "score": float(score)}) if self.wake.triggered(score): log.info(f"wake! score={score:.2f}") diff --git a/src/javis/api/models.py b/src/javis/api/models.py index 9e93742..6ee9aa2 100644 --- a/src/javis/api/models.py +++ b/src/javis/api/models.py @@ -11,6 +11,9 @@ class WakeWordBlock(BaseModel): threshold: float = 0.5 vad_threshold: float = 0.5 silence_ms: int = 800 + # How much louder than the background a frame must be to count as the user + # talking (0 = off). Older satellites simply ignore this field. + voice_gate_ratio: float = 3.0 class PairRequest(BaseModel): diff --git a/src/javis/api/routes.py b/src/javis/api/routes.py index 65337c2..d3e06b7 100644 --- a/src/javis/api/routes.py +++ b/src/javis/api/routes.py @@ -55,6 +55,7 @@ def _wake_word_block(config: Config) -> WakeWordBlock: threshold=ww.threshold, vad_threshold=ww.vad_threshold, silence_ms=ww.silence_ms, + voice_gate_ratio=ww.voice_gate_ratio, ) diff --git a/src/javis/audio_io/capture.py b/src/javis/audio_io/capture.py index 7124d3c..8c5799c 100644 --- a/src/javis/audio_io/capture.py +++ b/src/javis/audio_io/capture.py @@ -102,6 +102,7 @@ async def record_utterance( silent_run = 0 seen_speech = False chunks_seen = 0 + background = 0.0 # A vanished input device would otherwise block here forever while the # caller holds the turn lock. @@ -109,6 +110,7 @@ async def record_utterance( chunks_seen += 1 collected.append(chunk) result = vad.process(chunk) + background = result.background if result.is_speech: seen_speech = True @@ -117,9 +119,10 @@ async def record_utterance( silent_run += 1 if seen_speech and silent_run >= silence_chunks_needed: + log.debug(f"speech ended (background rms={background:.4f})") break if not seen_speech and chunks_seen >= grace_chunks: - log.debug("no speech during grace window") + log.debug(f"no speech during grace window (background rms={background:.4f})") return None if chunks_seen >= max_chunks: log.info("max utterance length reached") diff --git a/src/javis/audio_io/vad.py b/src/javis/audio_io/vad.py index fdffd5f..45b384e 100644 --- a/src/javis/audio_io/vad.py +++ b/src/javis/audio_io/vad.py @@ -1,8 +1,9 @@ """Voice activity detection using silero-vad (with graceful fallback).""" from __future__ import annotations +from collections import deque from dataclasses import dataclass -from typing import Protocol +from typing import Deque, Protocol import numpy as np @@ -15,6 +16,10 @@ log = get_logger("audio.vad") class VADResult: is_speech: bool probability: float + # Loudness of this frame and the background level it was judged against. + # Both are 0.0 for VADs that do not track loudness. + rms: float = 0.0 + background: float = 0.0 class VAD(Protocol): @@ -49,13 +54,125 @@ class NullVAD: if chunk.size == 0: return VADResult(False, 0.0) rms = float(np.sqrt(np.mean(np.square(chunk.astype(np.float32))))) - return VADResult(is_speech=rms >= self.threshold, probability=rms) + return VADResult(is_speech=rms >= self.threshold, probability=rms, rms=rms) -def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD: - """Construct a VAD, falling back to NullVAD if torch/silero fails to load.""" +# A frame must be at least this loud (RMS, normalized float32) to count as +# someone talking. Without the floor the gate would collapse to ~0 in a silent +# room and every faint noise would read as speech. +_MIN_VOICE_RMS = 0.006 +# Speech has short gaps, so a low percentile of the recent window still tracks +# the room even while the gate is open. +_BACKGROUND_PERCENTILE = 20.0 +# Once the user is already talking the gate drops to this fraction of the onset +# ratio, so a trailing quiet syllable is not clipped off mid-word. +_RELEASE_SCALE = 0.55 + + +def frame_rms(chunk: np.ndarray) -> float: + """RMS loudness of one mono float32 frame (0.0 for an empty frame).""" + if chunk is None or len(chunk) == 0: + return 0.0 + return float(np.sqrt(np.mean(np.square(np.asarray(chunk, dtype=np.float32))))) + + +class SpeechEndpointer: + """Reports only the *near-field* talker as speech. + + Silero answers "is this audio voice-like?", which music, TV and podcasts + satisfy just as well as the user does. When the device plays music through + its own speakers the VAD therefore never goes quiet, so a recording never + detects the end of the command and runs until the hard length cap. + + This wrapper adds the missing distinction: it tracks how loud the + surrounding audio is and only reports speech while a frame is clearly + louder than that background — which is exactly what separates someone + talking into the device from steady playback. Idle audio fed through + :meth:`observe` (the wake-word loops do) means the background is already + known by the time a recording starts. + + Implements the :class:`VAD` protocol, so it can be used anywhere a VAD is + expected. With ``onset_ratio <= 1.0`` the gate is disabled and the inner + VAD's verdict is passed through unchanged. + """ + + def __init__(self, vad: VAD, *, onset_ratio: float = 3.0, + window_s: float = 1.5, chunk_ms: int = 32) -> None: + self.vad = vad + self.sample_rate = vad.sample_rate + self.onset_ratio = max(0.0, float(onset_ratio)) + self.release_ratio = max(1.0, self.onset_ratio * _RELEASE_SCALE) + window = max(4, int(window_s * 1000.0 / max(1, chunk_ms))) + self._levels: Deque[float] = deque(maxlen=window) + self._in_speech = False + + @property + def threshold(self) -> float: + return self.vad.threshold + + @threshold.setter + def threshold(self, value: float) -> None: + self.vad.threshold = value + + @property + def enabled(self) -> bool: + """False when the loudness gate is off (plain VAD behaviour).""" + return self.onset_ratio > 1.0 + + @property + def background(self) -> float: + """Background loudness of the recent window; 0.0 until audio is seen.""" + if not self._levels: + return 0.0 + levels = sorted(self._levels) + idx = min(len(levels) - 1, + int(len(levels) * _BACKGROUND_PERCENTILE / 100.0)) + return levels[idx] + + def observe(self, chunk: np.ndarray) -> float: + """Feed audio heard while *not* recording; returns the frame's RMS. + + Anything heard outside a turn is background by definition, so the gate + is calibrated before the user even says the wake word. + """ + rms = frame_rms(chunk) + self._levels.append(rms) + self._in_speech = False + return rms + + def reset(self) -> None: + """Start a new utterance. Deliberately keeps the learned background.""" + self._in_speech = False + self.vad.reset() + + def process(self, chunk: np.ndarray) -> VADResult: + rms = frame_rms(chunk) + background = self.background + inner = self.vad.process(chunk) + if not self.enabled: + self._levels.append(rms) + return VADResult(inner.is_speech, inner.probability, rms, background) + ratio = self.release_ratio if self._in_speech else self.onset_ratio + gate = max(_MIN_VOICE_RMS, background * ratio) + is_voice = bool(inner.is_speech) and rms >= gate + self._in_speech = is_voice + if not is_voice: + # Only non-voice frames describe the background; letting speech in + # would let a long sentence raise the gate onto itself and cut the + # user off mid-utterance. + self._levels.append(rms) + return VADResult(is_voice, inner.probability, rms, background) + + +def make_vad(sample_rate: int = 16000, threshold: float = 0.5, + voice_gate_ratio: float = 3.0, chunk_ms: int = 32) -> VAD: + """Construct the capture VAD, falling back to NullVAD if torch/silero fails. + + The VAD is wrapped in a :class:`SpeechEndpointer` so that playback from the + device's own speakers is not mistaken for the user talking. + """ try: - return SileroVAD(sample_rate=sample_rate, threshold=threshold) + base: VAD = SileroVAD(sample_rate=sample_rate, threshold=threshold) except Exception as e: # noqa: BLE001 from ..core.hardware import detect_hardware, format_remediation log.error(f"SileroVAD unavailable: {e}") @@ -63,7 +180,8 @@ def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD: log.error(format_remediation(detect_hardware(), torch_error=str(e))) except Exception: pass - return NullVAD(sample_rate=sample_rate, threshold=threshold) + base = NullVAD(sample_rate=sample_rate, threshold=threshold) + return SpeechEndpointer(base, onset_ratio=voice_gate_ratio, chunk_ms=chunk_ms) class SileroVAD: diff --git a/src/javis/core/config.py b/src/javis/core/config.py index c6df7d6..82669b1 100644 --- a/src/javis/core/config.py +++ b/src/javis/core/config.py @@ -128,6 +128,11 @@ class WakeWordConfig: # VAD probability is above this value (cuts false wakes from TV/music). # 0 = off. vad_gate: float = 0.0 + # How much louder than the background a frame must be to count as the user + # talking. This is what lets a recording end when the *user* stops even + # while the device's own speakers keep playing music (which Silero happily + # classifies as speech). 0 = off (plain VAD endpointing). + voice_gate_ratio: float = 3.0 @dataclass @@ -425,6 +430,8 @@ def validate(cfg: "Config") -> None: cfg.speaker_id.refine_margin, 0.0, 0.5, 0.10, "speaker_id.refine_margin") cfg.wake_word.vad_gate = _clamp( cfg.wake_word.vad_gate, 0.0, 1.0, 0.0, "wake_word.vad_gate") + cfg.wake_word.voice_gate_ratio = _clamp( + cfg.wake_word.voice_gate_ratio, 0.0, 20.0, 3.0, "wake_word.voice_gate_ratio") # Silero VAD and openWakeWord are only wired up for 16 kHz input. if cfg.audio.sample_rate != 16000: diff --git a/tests/test_endpointing.py b/tests/test_endpointing.py new file mode 100644 index 0000000..aed4336 --- /dev/null +++ b/tests/test_endpointing.py @@ -0,0 +1,225 @@ +"""Utterance endpointing while the device plays audio through its own speakers. + +Silero rates music/TV as speech, so without a loudness gate a recording never +sees silence and runs until the hard length cap. These tests drive synthetic +frames through the real ``record_utterance`` loop. +""" +from __future__ import annotations + +import asyncio +import unittest + +import numpy as np + +from javis.audio_io.vad import SpeechEndpointer, VADResult, make_vad + +CHUNK_MS = 32 +SR = 16000 +SAMPLES = SR * CHUNK_MS // 1000 + +MUSIC_RMS = 0.02 +VOICE_RMS = 0.15 +QUIET_RMS = 0.0005 + + +def frame(rms: float) -> np.ndarray: + """A constant-amplitude frame whose RMS is exactly ``rms``.""" + return np.full(SAMPLES, rms, dtype=np.float32) + + +class AlwaysSpeech: + """Stand-in for Silero: calls everything voice-like, just as it does for + music, singing and TV dialogue.""" + + sample_rate = SR + threshold = 0.5 + + def __init__(self) -> None: + self.resets = 0 + + def reset(self) -> None: + self.resets += 1 + + def process(self, chunk: np.ndarray) -> VADResult: + return VADResult(is_speech=True, probability=1.0) + + +class FakeMic: + chunk_ms = CHUNK_MS + + def __init__(self, frames: list[np.ndarray]) -> None: + self._frames = frames + + def drain(self) -> None: + pass + + async def chunks(self, stale_timeout=None): + for f in self._frames: + yield f + await asyncio.sleep(0) + + +def endpointer(ratio: float = 3.0) -> SpeechEndpointer: + return SpeechEndpointer(AlwaysSpeech(), onset_ratio=ratio, chunk_ms=CHUNK_MS) + + +def record(frames, vad, *, silence_ms=800, grace_ms=1500, max_seconds=20.0): + from javis.audio_io.capture import record_utterance + from javis.core.config import Config + + cfg = Config() + cfg.wake_word.silence_ms = silence_ms + return asyncio.run(record_utterance( + cfg, FakeMic(frames), vad, max_seconds=max_seconds, + initial_grace_ms=grace_ms, + )) + + +class SpeechEndpointerTests(unittest.TestCase): + def test_music_alone_is_not_speech_once_background_is_known(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + ep.reset() + self.assertFalse(ep.process(frame(MUSIC_RMS)).is_speech) + + def test_voice_over_music_is_speech(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + ep.reset() + self.assertTrue(ep.process(frame(VOICE_RMS)).is_speech) + + def test_quiet_room_still_detects_normal_speech(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(QUIET_RMS)) + ep.reset() + self.assertTrue(ep.process(frame(0.05)).is_speech) + # ...but not the room tone itself. + ep.reset() + self.assertFalse(ep.process(frame(QUIET_RMS)).is_speech) + + def test_long_utterance_does_not_raise_the_gate_onto_itself(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(QUIET_RMS)) + ep.reset() + for _ in range(200): + self.assertTrue(ep.process(frame(0.05)).is_speech) + + def test_release_is_more_forgiving_than_onset(self) -> None: + """A trailing quiet syllable must not be clipped off mid-word.""" + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + ep.reset() + trailing = frame(MUSIC_RMS * 2.0) # below onset (3x), above release + self.assertFalse(ep.process(trailing).is_speech) + ep.process(frame(VOICE_RMS)) # user starts talking + self.assertTrue(ep.process(trailing).is_speech) + + def test_reset_keeps_the_background_but_clears_the_latch(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + ep.process(frame(VOICE_RMS)) + before = ep.background + ep.reset() + self.assertAlmostEqual(ep.background, before) + self.assertFalse(ep.process(frame(MUSIC_RMS * 2.0)).is_speech) + + def test_ratio_zero_disables_the_gate(self) -> None: + ep = endpointer(ratio=0.0) + self.assertFalse(ep.enabled) + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + ep.reset() + self.assertTrue(ep.process(frame(MUSIC_RMS)).is_speech) + + def test_reset_resets_the_inner_vad(self) -> None: + inner = AlwaysSpeech() + ep = SpeechEndpointer(inner, chunk_ms=CHUNK_MS) + ep.reset() + self.assertEqual(inner.resets, 1) + + def test_threshold_proxies_to_the_inner_vad(self) -> None: + inner = AlwaysSpeech() + ep = SpeechEndpointer(inner, chunk_ms=CHUNK_MS) + ep.threshold = 0.9 + self.assertEqual(inner.threshold, 0.9) + self.assertEqual(ep.threshold, 0.9) + + def test_empty_frame_is_harmless(self) -> None: + ep = endpointer() + self.assertEqual(ep.observe(np.zeros(0, dtype=np.float32)), 0.0) + self.assertFalse(ep.process(np.zeros(0, dtype=np.float32)).is_speech) + + +class RecordUtteranceEndpointingTests(unittest.TestCase): + def test_recording_ends_when_the_user_stops_not_the_music(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) # wake-word loop heard the music + + speech = [frame(VOICE_RMS)] * 30 # ~1.0 s of command + tail = [frame(MUSIC_RMS)] * 200 # music keeps playing + audio = record(speech + tail, ep) + + self.assertIsNotNone(audio) + # 30 speech frames + 25 frames (800 ms) of "silence" = ~1.76 s, far + # short of the 20 s cap the old endpointer ran into. + self.assertLess(len(audio) / SR, 3.0) + + def test_music_only_returns_none_instead_of_20s_of_audio(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(MUSIC_RMS)) + audio = record([frame(MUSIC_RMS)] * 400, ep) + self.assertIsNone(audio) + + def test_without_the_gate_music_runs_to_the_cap(self) -> None: + """The behaviour this change fixes, pinned so it cannot creep back.""" + ep = endpointer(ratio=0.0) + audio = record([frame(MUSIC_RMS)] * 400, ep, max_seconds=2.0) + self.assertIsNotNone(audio) + self.assertGreaterEqual(len(audio) / SR, 1.9) + + def test_quiet_room_recording_is_unaffected(self) -> None: + ep = endpointer() + for _ in range(50): + ep.observe(frame(QUIET_RMS)) + speech = [frame(0.05)] * 30 + tail = [frame(QUIET_RMS)] * 200 + audio = record(speech + tail, ep) + self.assertIsNotNone(audio) + self.assertLess(len(audio) / SR, 3.0) + + +class MakeVadWiringTests(unittest.TestCase): + def test_make_vad_returns_an_endpointer(self) -> None: + vad = make_vad(voice_gate_ratio=2.5) + self.assertIsInstance(vad, SpeechEndpointer) + self.assertEqual(vad.onset_ratio, 2.5) + self.assertTrue(hasattr(vad, "observe")) + + def test_config_clamps_the_ratio(self) -> None: + from javis.core.config import Config, validate + + cfg = Config() + self.assertEqual(cfg.wake_word.voice_gate_ratio, 3.0) + cfg.wake_word.voice_gate_ratio = -1.0 + validate(cfg) + self.assertEqual(cfg.wake_word.voice_gate_ratio, 3.0) + + def test_master_pushes_the_ratio_to_slaves(self) -> None: + from javis.api.routes import _wake_word_block + from javis.core.config import Config + + cfg = Config() + cfg.wake_word.voice_gate_ratio = 4.0 + self.assertEqual(_wake_word_block(cfg).voice_gate_ratio, 4.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_slave_audit.py b/tests/test_slave_audit.py index 2cd6a02..bff95f4 100644 --- a/tests/test_slave_audit.py +++ b/tests/test_slave_audit.py @@ -165,9 +165,74 @@ def test_send_turn_reads_master_followup_flag(): assert result.expects_followup is True +def test_slave_voice_gate_from_master_is_applied(monkeypatch): + """The master owns the loudness gate, so a satellite must honour the value + it pushes (and fall back to its own default for older masters).""" + from javis_slave.api_client import _wake_word_from + from javis_slave.audio_engine import AudioEngine + from javis_slave.config import SlaveConfig, WakeWordSettings + from javis_slave import audio_engine as ae + + assert _wake_word_from({"wake_word": {"voice_gate_ratio": 4.5}}).voice_gate_ratio == 4.5 + assert _wake_word_from({"wake_word": {}}).voice_gate_ratio == 3.0 + + built = [] + monkeypatch.setattr(ae, "make_vad", lambda **kw: built.append(kw) or object()) + + eng = AudioEngine(SlaveConfig(), client=None) + eng._apply_master_settings(None, WakeWordSettings(voice_gate_ratio=4.5)) + + assert built and built[-1]["voice_gate_ratio"] == 4.5 + assert eng.cfg.wake_word.voice_gate_ratio == 4.5 + + +def test_slave_endpointer_ends_recording_while_music_plays(): + """Silero calls music speech; without the loudness gate the satellite + recorded until the hard cap instead of when the user stopped talking.""" + import numpy as np + from javis_slave.audio_io.capture import record_utterance + from javis_slave.audio_io.vad import SpeechEndpointer, VADResult + + sr, chunk_ms = 16000, 32 + n = sr * chunk_ms // 1000 + + class AlwaysSpeech: + sample_rate = sr + threshold = 0.5 + + def reset(self): + pass + + def process(self, chunk): + return VADResult(True, 1.0) + + class FakeMic: + chunk_ms = 32 + + def __init__(self, frames): + self._frames = frames + + async def chunks(self, stale_timeout=None): + for f in self._frames: + yield f + await asyncio.sleep(0) + + def frame(rms): + return np.full(n, rms, dtype=np.float32) + + vad = SpeechEndpointer(AlwaysSpeech(), onset_ratio=3.0, chunk_ms=chunk_ms) + for _ in range(50): + vad.observe(frame(0.02)) # wake loop heard the music + + frames = [frame(0.15)] * 30 + [frame(0.02)] * 300 + audio = asyncio.run(record_utterance(sr, 800, FakeMic(frames), vad)) + + assert audio is not None + assert len(audio) / sr < 3.0 + + def test_slave_followup_skips_wake_word_and_starts_recording(): from javis_slave.audio_engine import AudioEngine - async def run(): eng = AudioEngine(SlaveConfig(), client=None) eng._followup_pending = True