From 74bc1be2d6ffc3b870da4224aff84842a8775d8d Mon Sep 17 00:00:00 2001 From: Javis Date: Sat, 10 Oct 2026 12:51:36 +0200 Subject: [PATCH] Start listening for expected follow-ups Propagate the master's follow-up decision to slave devices and skip wake-word detection for one turn after response playback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- javis-slave/README.md | 2 + javis-slave/src/javis_slave/api_client.py | 2 + javis-slave/src/javis_slave/audio_engine.py | 53 ++++++++++--------- src/javis/agent/orchestrator.py | 7 +++ src/javis/api/models.py | 3 ++ src/javis/api/routes.py | 1 + tests/test_slave_audit.py | 57 +++++++++++++++++++++ 7 files changed, 101 insertions(+), 24 deletions(-) diff --git a/javis-slave/README.md b/javis-slave/README.md index 19f0826..7763b2a 100644 --- a/javis-slave/README.md +++ b/javis-slave/README.md @@ -27,6 +27,8 @@ screen (GUI mode) and played back through the device's speakers. 4. **Turns** — After the wake word, the utterance is recorded locally (Silero VAD or RMS end-pointing), uploaded as WAV to `POST /api/v1/turn`, and the master's reply text + synthesized speech are returned and played. + When the master marks a reply as expecting a follow-up, the slave starts + listening immediately after playback without requiring another wake word. If the master's turn fails it returns its spoken error phrase instead, so a screenless satellite still gets audible feedback; connection problems play a low falling tone. diff --git a/javis-slave/src/javis_slave/api_client.py b/javis-slave/src/javis_slave/api_client.py index f571df5..89575c7 100644 --- a/javis-slave/src/javis_slave/api_client.py +++ b/javis-slave/src/javis_slave/api_client.py @@ -37,6 +37,7 @@ class TurnResult: audio_wav: Optional[bytes] # decoded WAV bytes (16-bit PCM) sample_rate: Optional[int] error: Optional[str] = None # master-side failure; reply_text is the spoken error phrase + expects_followup: bool = False def _http_detail(r: httpx.Response) -> str: @@ -263,6 +264,7 @@ class MasterClient: audio_wav=base64.b64decode(audio_b64) if audio_b64 else None, sample_rate=data.get("sample_rate"), error=data.get("error") or None, + expects_followup=bool(data.get("expects_followup", False)), ) async def check_health(self, host: str, port: int) -> bool: diff --git a/javis-slave/src/javis_slave/audio_engine.py b/javis-slave/src/javis_slave/audio_engine.py index bf00e78..fdeb5ba 100644 --- a/javis-slave/src/javis_slave/audio_engine.py +++ b/javis-slave/src/javis_slave/audio_engine.py @@ -72,8 +72,9 @@ class AudioEngine: self.wake: Optional[WakeWordDetector] = None self.vad = None self._config_task: Optional[asyncio.Task] = None - # Set when last response ended with '?'; next recording will use 3x silence timeout - self._last_response_is_question: bool = False + # Set by the master when the reply expects an answer. The next loop + # skips wake-word detection and starts recording after playback ends. + self._followup_pending: bool = False # UI callbacks (assigned by the app layer). self.on_state: Callable[[str], None] = lambda s: None @@ -203,21 +204,27 @@ class AudioEngine: await self._wait_unmuted_or_stop() continue try: - woke = await self._wait_for_wake() - if not woke: - if self._stop.is_set() or self.muted: + followup = self._followup_pending + if followup: + self._followup_pending = False + log.info("follow-up mode: skipping wake word") + spawn(self._play_chime([880.0, 1320.0], 0.06, 0.18)) + else: + woke = await self._wait_for_wake() + if not woke: + if self._stop.is_set() or self.muted: + continue + # The mic yielded nothing for _WAKE_STALE_S: USB device + # unplugged / driver stalled. Reopen instead of looping forever. + stale_exits += 1 + if stale_exits >= _MIC_RESTART_AFTER_STALE and self.mic is not None: + log.warning("mic silent for too long — reopening device") + self.on_error("Mikrofon liefert keine Daten — Gerät wird neu geöffnet …") + await self.mic.restart() + stale_exits = 0 continue - # The mic yielded nothing for _WAKE_STALE_S: USB device - # unplugged / driver stalled. Reopen instead of looping forever. - stale_exits += 1 - if stale_exits >= _MIC_RESTART_AFTER_STALE and self.mic is not None: - log.warning("mic silent for too long — reopening device") - self.on_error("Mikrofon liefert keine Daten — Gerät wird neu geöffnet …") - await self.mic.restart() - stale_exits = 0 - continue stale_exits = 0 - await self._handle_turn() + await self._handle_turn(is_followup=followup) consecutive_errors = 0 except asyncio.CancelledError: raise @@ -282,16 +289,17 @@ class AudioEngine: """Low descending two-tone: "something went wrong".""" await self._play_chime([440.0, 330.0], 0.12, 0.2) - async def _handle_turn(self) -> None: + async def _handle_turn(self, *, is_followup: bool = False) -> None: assert self.mic is not None self.on_state("listening") - # If last response was a question, triple the listening timeout silence_override = None - if self._last_response_is_question: + if is_followup: base_silence = self.cfg.wake_word.silence_ms silence_override = base_silence * 3 - log.debug(f"last response was a question; extending silence timeout to {silence_override}ms (was {base_silence}ms)") - self._last_response_is_question = False # reset flag for next turn + log.debug( + f"follow-up recording: extending silence timeout to " + f"{silence_override}ms (was {base_silence}ms)" + ) audio = await record_utterance( self.cfg.sample_rate, self.cfg.wake_word.silence_ms, self.mic, self.vad, silence_ms_override=silence_override, @@ -328,10 +336,6 @@ class AudioEngine: if result.user_text or result.reply_text: self.on_turn(result.user_text, result.reply_text) - # Check if response ended with '?' to trigger extended listening on next turn - if result.reply_text.rstrip().endswith("?"): - log.debug("assistant response ends with '?'; will extend listening timeout on next turn") - self._last_response_is_question = True if result.audio_wav and result.sample_rate: self.on_state("speaking") @@ -346,6 +350,7 @@ class AudioEngine: cancel_event=self._stop) except Exception as e: # noqa: BLE001 log.warning(f"reply playback failed: {e}") + self._followup_pending = result.expects_followup and not bool(result.error) self.on_state("idle") # ---------------- config refresh ---------------- diff --git a/src/javis/agent/orchestrator.py b/src/javis/agent/orchestrator.py index 40e0ec2..f7a3ada 100644 --- a/src/javis/agent/orchestrator.py +++ b/src/javis/agent/orchestrator.py @@ -350,6 +350,7 @@ class RemoteTurnResult: reply_text: str = "" audio: Optional[np.ndarray] = None # float32 mono reply speech sample_rate: Optional[int] = None + expects_followup: bool = False class Orchestrator: @@ -874,12 +875,17 @@ class Orchestrator: ) await run_start(self._middleware, turn_ctx) reply = "" + expects_followup = False try: reply = await self._llm_loop(turn_ctx, tts_feed=None) if reply: await self.bus.publish("assistant_text", reply) await run_finish(self._middleware, turn_ctx, text, reply) await turn_ctx.flush() + expects_followup = ( + self.config.wake_word.followup_enabled + and _reply_expects_followup(reply) + ) else: await run_abort(self._middleware, turn_ctx, "empty_reply") log.info(f"remote turn complete in {time.monotonic()-t0:.2f}s") @@ -899,6 +905,7 @@ class Orchestrator: reply_text=reply, audio=reply_audio, sample_rate=reply_sr, + expects_followup=expects_followup, ) async def synthesize_speech(self, text: str) -> tuple[Optional[np.ndarray], Optional[int]]: diff --git a/src/javis/api/models.py b/src/javis/api/models.py index b26a9c3..9e93742 100644 --- a/src/javis/api/models.py +++ b/src/javis/api/models.py @@ -35,6 +35,9 @@ class TurnResponse(BaseModel): reply_text: str = "" tts_audio_b64: Optional[str] = None # base64 WAV (16-bit PCM mono) sample_rate: Optional[int] = None + # When true, the slave listens for one answer immediately after playback + # instead of requiring another wake word. + expects_followup: bool = False # Set when the master could not complete the turn. reply_text then holds # the spoken error phrase (and tts_audio_b64 its audio) so a screenless # slave can still tell the user something went wrong. diff --git a/src/javis/api/routes.py b/src/javis/api/routes.py index f61ec8f..65337c2 100644 --- a/src/javis/api/routes.py +++ b/src/javis/api/routes.py @@ -203,6 +203,7 @@ async def turn( reply_text=result.reply_text, tts_audio_b64=_wav_b64(result.audio, result.sample_rate), sample_rate=result.sample_rate if result.audio is not None else None, + expects_followup=result.expects_followup, ) diff --git a/tests/test_slave_audit.py b/tests/test_slave_audit.py index 052a6bc..2cd6a02 100644 --- a/tests/test_slave_audit.py +++ b/tests/test_slave_audit.py @@ -143,6 +143,63 @@ def test_send_turn_retries_connect_error_once(): assert len(calls) == 2 +def test_send_turn_reads_master_followup_flag(): + def handler(request): + return httpx.Response( + 200, + json={ + "user_text": "Ja", + "reply_text": "Noch etwas?", + "expects_followup": True, + }, + ) + + async def run(): + client = _client_with(handler) + try: + return await client.send_turn(b"RIFF") + finally: + await client.aclose() + + result = asyncio.run(run()) + assert result.expects_followup is True + + +def test_slave_followup_skips_wake_word_and_starts_recording(): + from javis_slave.audio_engine import AudioEngine + + async def run(): + eng = AudioEngine(SlaveConfig(), client=None) + eng._followup_pending = True + calls = [] + + async def config_poll(): + await eng._stop.wait() + + async def wait_for_wake(): + calls.append("wake") + return True + + async def handle_turn(*, is_followup=False): + calls.append(("turn", is_followup)) + eng._stop.set() + + async def play_chime(*args): + calls.append("chime") + + eng._config_poll_loop = config_poll + eng._wait_for_wake = wait_for_wake + eng._handle_turn = handle_turn + eng._play_chime = play_chime + await eng.run() + await asyncio.sleep(0) + return calls + + calls = asyncio.run(run()) + assert "wake" not in calls + assert ("turn", True) in calls + + # ── SLV-04 / SLV-06: audio engine ──────────────────────────────────────────── def test_engine_start_failure_releases_mic(monkeypatch):