End recording when the user stops, not the music

Silero VAD answers "is this voice-like?", which music, TV and podcasts
satisfy just as well as the user does. With Spotify playing through the
device's own speakers the VAD never went quiet, so record_utterance never
saw the end of the command and ran to the 20 s cap on master and slave
alike -- and music alone was enough to defeat the "no speech during the
grace window" early return.

Wrap the VAD in a SpeechEndpointer that tracks the background loudness of
the recent window and only reports speech while a frame is clearly louder
than it. Both wake-word loops feed their idle audio through observe(), so
the gate is already calibrated when a turn starts, and only non-voice
frames update the background so a long sentence cannot raise the gate onto
itself. Onset/release hysteresis keeps trailing quiet syllables.

The master owns the knob (wake_word.voice_gate_ratio, 0 = off) and pushes
it to satellites in the wake-word block.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

javis-slave/README.md

@@ -27,6 +27,11 @@ screen (GUI mode) and played back through the device's speakers.
 4. **Turns** — After the wake word, the utterance is recorded locally
    (Silero VAD or RMS end-pointing), uploaded as WAV to `POST /api/v1/turn`,
    and the master's reply text + synthesized speech are returned and played.
+   Recording ends when *your voice* stops: Silero rates music and TV as
+   speech, so a loudness gate (`wake_word.voice_gate_ratio`, pushed by the
+   master) only counts audio that is clearly louder than the background the
+   wake-word loop measured. Without it, music from this device's own speakers
+   kept the recorder running until the hard length cap.
    When the master marks a reply as expecting a follow-up, the slave starts
    listening immediately after playback without requiring another wake word.
    If the master's turn fails it returns its spoken error phrase instead, so a

javis-slave/src/javis_slave/api_client.py

@@ -58,6 +58,8 @@ def _wake_word_from(payload: dict) -> WakeWordSettings:
         threshold=float(ww.get("threshold", 0.5)),
         vad_threshold=float(ww.get("vad_threshold", 0.5)),
         silence_ms=int(ww.get("silence_ms", 800)),
+        # Older masters do not send this; keep the satellite's own default.
+        voice_gate_ratio=float(ww.get("voice_gate_ratio", 3.0)),
     )
 
 

javis-slave/src/javis_slave/audio_engine.py

@@ -104,7 +104,10 @@ class AudioEngine:
         )
         try:
             await self.mic.start()
-            self.vad = make_vad(threshold=self.cfg.wake_word.vad_threshold)
+            self.vad = make_vad(
+                threshold=self.cfg.wake_word.vad_threshold,
+                voice_gate_ratio=self.cfg.wake_word.voice_gate_ratio,
+            )
             self.wake = await self._load_wake(self.cfg.wake_word.model, self.cfg.wake_word.threshold)
         except BaseException:
             # The caller retries with a fresh engine; an open stream left behind
@@ -261,10 +264,14 @@ class AudioEngine:
                 self.on_state("muted")
                 return False
             score = self.wake.feed(chunk)
+            # Everything heard before the wake word is background (music, TV,
+            # this device's own speakers), so the recorder's voice gate is
+            # already calibrated when the turn starts.
+            rms = self.vad.observe(chunk) if hasattr(self.vad, "observe") else (
+                float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0)
             now = time.monotonic()
             if now - last_pub > 0.1:
                 last_pub = now
-                rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0
                 self.last_rms = rms
                 self.last_score = float(score)
                 self.on_mic_level(rms, float(score))
@@ -386,8 +393,8 @@ class AudioEngine:
             self.on_room(room)
             changed = True
         old = self.cfg.wake_word
-        if (ww.model, ww.threshold, ww.vad_threshold, ww.silence_ms) != (
-                old.model, old.threshold, old.vad_threshold, old.silence_ms):
+        if (ww.model, ww.threshold, ww.vad_threshold, ww.silence_ms, ww.voice_gate_ratio) != (
+                old.model, old.threshold, old.vad_threshold, old.silence_ms, old.voice_gate_ratio):
             log.info(f"wake-word settings updated by master: {ww}")
             rebuild_model = ww.model != old.model
             self.cfg.wake_word = ww
@@ -397,7 +404,8 @@ class AudioEngine:
                     spawn(self._swap_wake_model(ww.model, ww.threshold))
                 else:
                     self.wake.threshold = ww.threshold
-            self.vad = make_vad(threshold=ww.vad_threshold)
+            self.vad = make_vad(
+                threshold=ww.vad_threshold, voice_gate_ratio=ww.voice_gate_ratio)
             changed = True
         return changed
 

javis-slave/src/javis_slave/audio_io/capture.py

@@ -57,11 +57,13 @@ async def record_utterance(
     silent_run = 0
     seen_speech = False
     chunks_seen = 0
+    background = 0.0
 
     async for chunk in mic.chunks(stale_timeout=stale_timeout):
         chunks_seen += 1
         collected.append(chunk)
         result = vad.process(chunk)
+        background = result.background
 
         if result.is_speech:
             seen_speech = True
@@ -70,9 +72,10 @@ async def record_utterance(
             silent_run += 1
 
         if seen_speech and silent_run >= silence_chunks_needed:
+            log.debug(f"speech ended (background rms={background:.4f})")
             break
         if not seen_speech and chunks_seen >= grace_chunks:
-            log.debug("no speech during grace window")
+            log.debug(f"no speech during grace window (background rms={background:.4f})")
             return None
         if chunks_seen >= max_chunks:
             log.info("max utterance length reached")

javis-slave/src/javis_slave/audio_io/vad.py

@@ -5,8 +5,9 @@ diagnostics helper so this module has no dependency on the ``javis`` package.
 """
 from __future__ import annotations
 
+from collections import deque
 from dataclasses import dataclass
-from typing import Protocol
+from typing import Deque, Protocol
 
 import numpy as np
 
@@ -19,6 +20,10 @@ log = get_logger("audio.vad")
 class VADResult:
     is_speech: bool
     probability: float
+    # Loudness of this frame and the background level it was judged against.
+    # Both are 0.0 for VADs that do not track loudness.
+    rms: float = 0.0
+    background: float = 0.0
 
 
 class VAD(Protocol):
@@ -49,16 +54,129 @@ class NullVAD:
         if chunk.size == 0:
             return VADResult(False, 0.0)
         rms = float(np.sqrt(np.mean(np.square(chunk.astype(np.float32)))))
-        return VADResult(is_speech=rms >= self.threshold, probability=rms)
+        return VADResult(is_speech=rms >= self.threshold, probability=rms, rms=rms)
 
 
-def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD:
-    """Construct a VAD, falling back to NullVAD if torch/silero fails to load."""
+# A frame must be at least this loud (RMS, normalized float32) to count as
+# someone talking.  Without the floor the gate would collapse to ~0 in a silent
+# room and every faint noise would read as speech.
+_MIN_VOICE_RMS = 0.006
+# Speech has short gaps, so a low percentile of the recent window still tracks
+# the room even while the gate is open.
+_BACKGROUND_PERCENTILE = 20.0
+# Once the user is already talking the gate drops to this fraction of the onset
+# ratio, so a trailing quiet syllable is not clipped off mid-word.
+_RELEASE_SCALE = 0.55
+
+
+def frame_rms(chunk: np.ndarray) -> float:
+    """RMS loudness of one mono float32 frame (0.0 for an empty frame)."""
+    if chunk is None or len(chunk) == 0:
+        return 0.0
+    return float(np.sqrt(np.mean(np.square(np.asarray(chunk, dtype=np.float32)))))
+
+
+class SpeechEndpointer:
+    """Reports only the *near-field* talker as speech.
+
+    Silero answers "is this audio voice-like?", which music, TV and podcasts
+    satisfy just as well as the user does.  When the satellite plays music
+    through its own speakers the VAD therefore never goes quiet, so a recording
+    never detects the end of the command and runs until the hard length cap.
+
+    This wrapper adds the missing distinction: it tracks how loud the
+    surrounding audio is and only reports speech while a frame is clearly
+    louder than that background — which is exactly what separates someone
+    talking into the device from steady playback.  Idle audio fed through
+    :meth:`observe` (the wake-word loop does) means the background is already
+    known by the time a recording starts.
+
+    Implements the :class:`VAD` protocol, so it can be used anywhere a VAD is
+    expected.  With ``onset_ratio <= 1.0`` the gate is disabled and the inner
+    VAD's verdict is passed through unchanged.
+    """
+
+    def __init__(self, vad: VAD, *, onset_ratio: float = 3.0,
+                 window_s: float = 1.5, chunk_ms: int = 32) -> None:
+        self.vad = vad
+        self.sample_rate = vad.sample_rate
+        self.onset_ratio = max(0.0, float(onset_ratio))
+        self.release_ratio = max(1.0, self.onset_ratio * _RELEASE_SCALE)
+        window = max(4, int(window_s * 1000.0 / max(1, chunk_ms)))
+        self._levels: Deque[float] = deque(maxlen=window)
+        self._in_speech = False
+
+    @property
+    def threshold(self) -> float:
+        return self.vad.threshold
+
+    @threshold.setter
+    def threshold(self, value: float) -> None:
+        self.vad.threshold = value
+
+    @property
+    def enabled(self) -> bool:
+        """False when the loudness gate is off (plain VAD behaviour)."""
+        return self.onset_ratio > 1.0
+
+    @property
+    def background(self) -> float:
+        """Background loudness of the recent window; 0.0 until audio is seen."""
+        if not self._levels:
+            return 0.0
+        levels = sorted(self._levels)
+        idx = min(len(levels) - 1,
+                  int(len(levels) * _BACKGROUND_PERCENTILE / 100.0))
+        return levels[idx]
+
+    def observe(self, chunk: np.ndarray) -> float:
+        """Feed audio heard while *not* recording; returns the frame's RMS.
+
+        Anything heard outside a turn is background by definition, so the gate
+        is calibrated before the user even says the wake word.
+        """
+        rms = frame_rms(chunk)
+        self._levels.append(rms)
+        self._in_speech = False
+        return rms
+
+    def reset(self) -> None:
+        """Start a new utterance.  Deliberately keeps the learned background."""
+        self._in_speech = False
+        self.vad.reset()
+
+    def process(self, chunk: np.ndarray) -> VADResult:
+        rms = frame_rms(chunk)
+        background = self.background
+        inner = self.vad.process(chunk)
+        if not self.enabled:
+            self._levels.append(rms)
+            return VADResult(inner.is_speech, inner.probability, rms, background)
+        ratio = self.release_ratio if self._in_speech else self.onset_ratio
+        gate = max(_MIN_VOICE_RMS, background * ratio)
+        is_voice = bool(inner.is_speech) and rms >= gate
+        self._in_speech = is_voice
+        if not is_voice:
+            # Only non-voice frames describe the background; letting speech in
+            # would let a long sentence raise the gate onto itself and cut the
+            # user off mid-utterance.
+            self._levels.append(rms)
+        return VADResult(is_voice, inner.probability, rms, background)
+
+
+def make_vad(sample_rate: int = 16000, threshold: float = 0.5,
+             voice_gate_ratio: float = 3.0, chunk_ms: int = 32) -> VAD:
+    """Construct the capture VAD, falling back to NullVAD if torch/silero fails.
+
+    The VAD is wrapped in a :class:`SpeechEndpointer` so that playback from the
+    satellite's own speakers is not mistaken for the user talking.
+    """
     try:
-        return SileroVAD(sample_rate=sample_rate, threshold=threshold)
+        base: VAD = SileroVAD(sample_rate=sample_rate, threshold=threshold)
     except Exception as e:  # noqa: BLE001
         log.error(f"SileroVAD unavailable: {e}")
-        return NullVAD(sample_rate=sample_rate, threshold=threshold)
+        base = NullVAD(sample_rate=sample_rate, threshold=threshold)
+    return SpeechEndpointer(base, onset_ratio=voice_gate_ratio, chunk_ms=chunk_ms)
 
 
 class SileroVAD:

javis-slave/src/javis_slave/config.py

@@ -40,6 +40,10 @@ class WakeWordSettings:
     threshold: float = 0.5
     vad_threshold: float = 0.5
     silence_ms: int = 800
+    # How much louder than the background a frame must be to count as the user
+    # talking, so recording ends when the user stops even while this device's
+    # own speakers keep playing.  0 = off.
+    voice_gate_ratio: float = 3.0
 
 
 @dataclass