From b921938970549d336d30f93aab2d06cdfe9337ff Mon Sep 17 00:00:00 2001 From: Javis Date: Sat, 10 Oct 2026 12:46:59 +0200 Subject: [PATCH] Handle CUDA context loss gracefully Fall back to CPU for speaker identification and fail speech recognition with an actionable restart message when the NVIDIA driver resets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- config.example.yaml | 2 + src/javis/agent/orchestrator.py | 34 +++++-- src/javis/api/routes.py | 7 +- src/javis/core/config.py | 6 ++ src/javis/core/hardware.py | 14 +++ src/javis/stt/transcriber.py | 50 +++++++++-- src/javis/stt/voice_profiler.py | 53 ++++++++--- tests/test_gpu_lost.py | 152 ++++++++++++++++++++++++++++++++ 8 files changed, 293 insertions(+), 25 deletions(-) create mode 100644 tests/test_gpu_lost.py diff --git a/config.example.yaml b/config.example.yaml index 4785410..fe30e62 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -84,6 +84,8 @@ agent: max_llm_iterations: 5 # Spoken feedback for voice users (empty string = silent). error_phrase: "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal." + # Spoken when the GPU was lost (e.g. NVIDIA driver updated while running). + gpu_lost_phrase: "Die Grafikkarte ist nicht mehr erreichbar, vermutlich wurde der Treiber aktualisiert. Bitte starte Javis neu." approval_phrase: "Soll ich das ausführen? Sag ja oder nein." memory: diff --git a/src/javis/agent/orchestrator.py b/src/javis/agent/orchestrator.py index 087e5f9..40e0ec2 100644 --- a/src/javis/agent/orchestrator.py +++ b/src/javis/agent/orchestrator.py @@ -18,6 +18,7 @@ from ..core.config import Config from ..core.events import EventBus from ..core.logger import get_logger from ..core.state import AgentState +from ..core.hardware import GpuUnavailableError from ..core.tasks import cancel_and_wait, spawn from ..llm.conversation import Conversation from ..llm.hermes_client import HermesClient @@ -54,6 +55,9 @@ log = get_logger("agent") SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$") # Detects whether the last sentence of a reply is a question. +# Returned by Orchestrator._identify_speaker when speaker ID raised. +_ID_FAILED = object() + _FOLLOWUP_RE = re.compile(r'\?\s*$') @@ -658,10 +662,16 @@ class Orchestrator: raise except Exception as e: # noqa: BLE001 log.exception(f"turn failed: {e}") - await self.bus.publish("error", {"where": "turn", "message": str(e)}) + gpu_lost = isinstance(e, GpuUnavailableError) + await self.bus.publish( + "error", {"where": "stt" if gpu_lost else "turn", "message": str(e)} + ) await self.set_state(AgentState.ERROR) # Voice users can't see the GUI error — say something. - await self._speak_notice(self.config.agent.error_phrase) + await self._speak_notice( + self.config.agent.gpu_lost_phrase if gpu_lost + else self.config.agent.error_phrase + ) await asyncio.sleep(1.0) # Recover to IDLE so the agent can accept new input after an error. # Without this recovery, a single failure would leave Javis stuck @@ -797,7 +807,7 @@ class Orchestrator: if self.config.speaker_id.enabled: tr, id_result = await asyncio.gather( self.stt.transcribe(audio, sample_rate), - asyncio.to_thread(self.profiler.identify, audio, sample_rate), + self._identify_speaker(audio, sample_rate), ) else: tr = await self.stt.transcribe(audio, sample_rate) @@ -815,7 +825,9 @@ class Orchestrator: _sp_name: Optional[str] = None _privileges: SpeakerPrivileges = SpeakerPrivileges() # default: full access if self.config.speaker_id.enabled: - if id_result: + if id_result is _ID_FAILED: + _privileges = self._unknown_privileges() # fail closed, no enroll prompt + elif id_result: sp_name, sp_conf = id_result _sp_name = sp_name speaker_prefix = f"[Speaker: {sp_name}] " @@ -1243,6 +1255,14 @@ class Orchestrator: # Reset to IDLE so the GUI doesn't stay stuck on "listening" # when the turn ends without any speech. await self.set_state(AgentState.IDLE) + async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any: + """Run speaker ID off-loop; a failure must never abort the turn.""" + try: + return await asyncio.to_thread(self.profiler.identify, audio, sample_rate) + except Exception as exc: # noqa: BLE001 + log.exception(f"[profiler] speaker identification failed: {exc}") + return _ID_FAILED + return await self.set_state(AgentState.THINKING) @@ -1258,7 +1278,7 @@ class Orchestrator: if self.config.speaker_id.enabled: tr, id_result = await asyncio.gather( self.stt.transcribe(audio, self.config.audio.sample_rate), - asyncio.to_thread(self.profiler.identify, audio, self.config.audio.sample_rate), + self._identify_speaker(audio, self.config.audio.sample_rate), ) else: tr = await self.stt.transcribe(audio, self.config.audio.sample_rate) @@ -1273,7 +1293,9 @@ class Orchestrator: _sp_name: Optional[str] = None _privileges: SpeakerPrivileges = SpeakerPrivileges() # default: full access if self.config.speaker_id.enabled: - if id_result: + if id_result is _ID_FAILED: + _privileges = self._unknown_privileges() # fail closed, no enroll prompt + elif id_result: sp_name, sp_conf = id_result _sp_name = sp_name speaker_prefix = f"[Speaker: {sp_name}] " diff --git a/src/javis/api/routes.py b/src/javis/api/routes.py index 7b75ac4..f61ec8f 100644 --- a/src/javis/api/routes.py +++ b/src/javis/api/routes.py @@ -16,6 +16,7 @@ import numpy as np from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile from ..core.config import Config +from ..core.hardware import GpuUnavailableError from ..core.logger import get_logger from .devices import SlaveDevice, SlaveDeviceStore from .models import ConfigResponse, PairRequest, PairResponse, TurnResponse, WakeWordBlock, SlaveDeviceInfo @@ -183,7 +184,11 @@ async def turn( except Exception as exc: # noqa: BLE001 # Return a spoken error instead of a bare 500 so the slave can voice it. log.exception(f"remote turn from {device.name!r} failed: {exc}") - phrase = config.agent.error_phrase + phrase = ( + config.agent.gpu_lost_phrase + if isinstance(exc, GpuUnavailableError) + else config.agent.error_phrase + ) err_audio, err_sr = await orch.synthesize_speech(phrase) return TurnResponse( user_text="", diff --git a/src/javis/core/config.py b/src/javis/core/config.py index 9e6278a..c6df7d6 100644 --- a/src/javis/core/config.py +++ b/src/javis/core/config.py @@ -173,6 +173,12 @@ class AgentConfig: # Spoken when a voice turn fails, so users away from the screen get # feedback instead of silence. Empty string disables. error_phrase: str = "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal." + # Spoken instead of error_phrase when the GPU's CUDA context died (e.g. + # an NVIDIA driver update while Javis was running) — needs a restart. + gpu_lost_phrase: str = ( + "Die Grafikkarte ist nicht mehr erreichbar, vermutlich wurde der Treiber " + "aktualisiert. Bitte starte Javis neu." + ) # Spoken when a tool call is waiting for approval and the window is not # in the foreground. Empty string disables. approval_phrase: str = "Soll ich das ausführen? Sag ja oder nein." diff --git a/src/javis/core/hardware.py b/src/javis/core/hardware.py index 5b2e828..e2ce587 100644 --- a/src/javis/core/hardware.py +++ b/src/javis/core/hardware.py @@ -33,6 +33,20 @@ TORCH_INDEX_ROCM = "https://download.pytorch.org/whl/rocm6.0" CTRANSLATE2_REQUIRED_CUDA_DLL = "cublas64_12.dll" +class GpuUnavailableError(RuntimeError): + """The CUDA context died while Javis was running (typically an NVIDIA + driver update or reset). It cannot be recovered in-process — restart Javis.""" + + +def is_cuda_error(exc: BaseException) -> bool: + """True for CUDA / cuBLAS / cuDNN failures raised by torch or CTranslate2. + + Out-of-memory is excluded: it is transient and the context stays usable. + """ + msg = str(exc).lower() + return "out of memory" not in msg and any(k in msg for k in ("cuda", "cublas", "cudnn")) + + @dataclass class HardwareInfo: os_name: str = "" # 'Windows', 'Linux', 'Darwin' diff --git a/src/javis/stt/transcriber.py b/src/javis/stt/transcriber.py index 4c5d566..33a6c69 100644 --- a/src/javis/stt/transcriber.py +++ b/src/javis/stt/transcriber.py @@ -12,8 +12,14 @@ from ..core.logger import get_logger log = get_logger("stt") +from ..core.hardware import GpuUnavailableError, is_cuda_error from ..core.hardware import add_nvidia_dll_dirs as _add_nvidia_dll_dirs +_GPU_LOST_MSG = ( + "GPU (CUDA) lost — the NVIDIA driver was probably updated or reset while " + "Javis was running. Restart Javis to restore speech recognition." +) + # Register NVIDIA pip-package DLL dirs on PATH as early as possible so that # ctranslate2's internal LoadLibrary calls find cublas64_12.dll etc. before # the first CUDA operation is attempted. @@ -71,6 +77,22 @@ class Transcriber: self._model = None self._batched = None self._force_cpu: bool = False # set True after a CUDA runtime DLL failure + self._device: str = "" + # Set once the CUDA context died; every later call fails fast. + self._gpu_lost: bool = False + + @staticmethod + def _cuda_alive() -> bool: + """Probe the CUDA context with a tiny host→device copy.""" + try: + import ctranslate2 + ctranslate2.StorageView.from_array(np.zeros(1, np.float32)).to_device( + ctranslate2.Device.cuda + ) + return True + except Exception as exc: # noqa: BLE001 + log.debug(f"STT: CUDA probe failed: {exc}") + return "out of memory" in str(exc).lower() def _resolve_device_and_compute(self) -> tuple[str, str]: """Resolve the best device + compute_type for CTranslate2 / faster-whisper. @@ -173,6 +195,7 @@ class Transcriber: cpu_threads=0, # 0 = use all available cores download_root=str(self.config.paths.models_dir / "whisper"), ) + self._device = device # BatchedInferencePipeline (faster-whisper >=1.1) runs transcription in # parallel across the batch dimension — roughly 4x faster on GPU with # no meaningful WER change. CPU path skips this since batching adds @@ -196,6 +219,8 @@ class Transcriber: def _transcribe_sync(self, audio: np.ndarray, sample_rate: int) -> TranscriptionResult: if self._model is None: raise RuntimeError("Transcriber not loaded — call load() first") + if self._gpu_lost: + raise GpuUnavailableError(_GPU_LOST_MSG) if sample_rate != 16000: # resample_poly is O(N) and ~100x faster than scipy.signal.resample (FFT-based). from fractions import Fraction @@ -218,20 +243,30 @@ class Transcriber: def _run_transcribe() -> tuple: if self._batched is not None: - return self._batched.transcribe( + segments, info = self._batched.transcribe( audio, batch_size=self.config.stt.batch_size, **transcribe_kwargs, ) - return self._model.transcribe( - audio, - vad_filter=False, # VAD already applied upstream - **transcribe_kwargs, - ) + else: + segments, info = self._model.transcribe( + audio, + vad_filter=False, # VAD already applied upstream + **transcribe_kwargs, + ) + # Segments are a lazy generator — decode here so errors raised + # mid-decode reach the handler below. + return list(segments), info try: segments, info = _run_transcribe() except RuntimeError as exc: + if self._device == "cuda" and is_cuda_error(exc) and not self._cuda_alive(): + # Never free or reload the model here: destroying a CTranslate2 + # CUDA model on a dead context aborts the whole process. + self._gpu_lost = True + log.error(f"STT: {_GPU_LOST_MSG} ({exc})") + raise GpuUnavailableError(_GPU_LOST_MSG) from exc _msg = str(exc) if any(k in _msg.lower() for k in ("dll", "cublas", "cannot be loaded")): log.warning( @@ -245,8 +280,7 @@ class Transcriber: else: raise - segments = list(segments) - text = "".join(s.text for s in segments).strip() + text = "".join(s.text for s in segments).strip() return TranscriptionResult( text=text, language=info.language, diff --git a/src/javis/stt/voice_profiler.py b/src/javis/stt/voice_profiler.py index 5c1232c..b1abc74 100644 --- a/src/javis/stt/voice_profiler.py +++ b/src/javis/stt/voice_profiler.py @@ -13,6 +13,7 @@ available so the rest of the agent continues to work unchanged. from __future__ import annotations import json +import threading import time from dataclasses import dataclass, field from fnmatch import fnmatch as _fnmatch @@ -22,6 +23,7 @@ from typing import Optional, Union import numpy as np from ..core.config import Config +from ..core.hardware import is_cuda_error from ..core.logger import get_logger log = get_logger("voice_profiler") @@ -185,6 +187,8 @@ class VoiceProfiler: category=UserWarning, ) # 3. speechbrain Pretrainer symlinks — harmless on Windows because + self._device = "cpu" # device the encoder runs on + self._encoder_lock = threading.Lock() # serialises the CPU fallback # model files are copied rather than symlinked. warnings.filterwarnings( "ignore", @@ -192,18 +196,12 @@ class VoiceProfiler: category=UserWarning, ) - from speechbrain.inference.classifiers import EncoderClassifier import torch # noqa: F401 — verify torch present # SpeechBrain's run_opts parser expects "cuda:N", not bare "cuda". device = "cuda:0" if self._try_cuda() else "cpu" - savedir = str(self.config.paths.models_dir / "ecapa-tdnn") - log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}") - self._encoder = EncoderClassifier.from_hparams( - source=_MODEL_SOURCE, - savedir=savedir, - run_opts={"device": device}, - ) + self._encoder = self._build_encoder(device) + self._device = device log.info("[profiler] ECAPA-TDNN loaded") except Exception as exc: log.warning( @@ -256,6 +254,33 @@ class VoiceProfiler: # ── embedding ───────────────────────────────────────────────────────────── + def _build_encoder(self, device: str): + from speechbrain.inference.classifiers import EncoderClassifier + + savedir = str(self.config.paths.models_dir / "ecapa-tdnn") + log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}") + return EncoderClassifier.from_hparams( + source=_MODEL_SOURCE, + savedir=savedir, + run_opts={"device": device}, + ) + + def _fallback_to_cpu(self, failed, exc: BaseException): + """Swap a broken CUDA encoder for a CPU one (once, thread-safe). + + CUDA errors here usually mean the context died (e.g. the NVIDIA driver + was updated while Javis was running), which never recovers in-process. + """ + with self._encoder_lock: + if self._encoder is failed: + log.warning( + f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; " + "restart Javis to use the GPU again" + ) + self._encoder = self._build_encoder("cpu") + self._device = "cpu" + return self._encoder + def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray: """Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*.""" if self._encoder is None: @@ -284,8 +309,16 @@ class VoiceProfiler: wav = torch.tensor(audio_np.astype(np.float32)).unsqueeze(0) # [1, T] wav_lens = torch.tensor([1.0]) - with torch.no_grad(): - emb = self._encoder.encode_batch(wav, wav_lens) # [1, 1, D] + encoder = self._encoder + try: + with torch.no_grad(): + emb = encoder.encode_batch(wav, wav_lens) # [1, 1, D] + except Exception as exc: + if not (self._device.startswith("cuda") and is_cuda_error(exc)): + raise + encoder = self._fallback_to_cpu(encoder, exc) + with torch.no_grad(): + emb = encoder.encode_batch(wav, wav_lens) return _l2(emb.squeeze().cpu().numpy()) # ── CRUD ────────────────────────────────────────────────────────────────── diff --git a/tests/test_gpu_lost.py b/tests/test_gpu_lost.py new file mode 100644 index 0000000..3a99221 --- /dev/null +++ b/tests/test_gpu_lost.py @@ -0,0 +1,152 @@ +"""Surviving a dead CUDA context (e.g. NVIDIA driver updated while running).""" +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np + +from javis.core.config import Config, PathsConfig +from javis.core.hardware import GpuUnavailableError, is_cuda_error +from javis.stt.transcriber import Transcriber +from javis.stt.voice_profiler import VoiceProfiler + +try: + import torch +except ImportError: # pragma: no cover + torch = None + +SR = 16000 +_AUDIO = np.full(SR, 0.1, dtype=np.float32) + + +class IsCudaErrorTests(unittest.TestCase): + def test_classification(self) -> None: + self.assertTrue(is_cuda_error(RuntimeError("CUDA error: unknown error"))) + self.assertTrue(is_cuda_error(RuntimeError("CUDA failed with error unknown error"))) + self.assertTrue(is_cuda_error(RuntimeError("cuBLAS failed with status X"))) + self.assertFalse(is_cuda_error(RuntimeError("CUDA failed with error out of memory"))) + self.assertFalse(is_cuda_error(ValueError("bad input"))) + + +class _DeadModel: + """Fake WhisperModel whose lazy segment generator hits a dead context.""" + + def __init__(self, msg: str = "CUDA failed with error unknown error") -> None: + self.msg = msg + self.calls = 0 + + def transcribe(self, audio, **kw): + self.calls += 1 + + def segments(): + raise RuntimeError(self.msg) + yield # pragma: no cover + + return segments(), SimpleNamespace(language="de", duration=1.0) + + +def _transcriber(model) -> Transcriber: + stt = Transcriber(Config()) + stt._model = model + stt._batched = None + stt._device = "cuda" + return stt + + +class TranscriberGpuLostTests(unittest.TestCase): + def test_dead_context_raises_and_then_fails_fast(self) -> None: + model = _DeadModel() + stt = _transcriber(model) + with patch.object(Transcriber, "_cuda_alive", return_value=False): + with self.assertRaisesRegex(GpuUnavailableError, "Restart Javis"): + stt._transcribe_sync(_AUDIO, SR) + with self.assertRaises(GpuUnavailableError): + stt._transcribe_sync(_AUDIO, SR) + self.assertEqual(model.calls, 1) # second call never touched the GPU + self.assertIs(stt._model, model) # freeing it would abort the process + self.assertFalse(stt._force_cpu) + + def test_cuda_error_on_live_context_is_not_gpu_lost(self) -> None: + stt = _transcriber(_DeadModel("CUDA failed with error invalid argument")) + with patch.object(Transcriber, "_cuda_alive", return_value=True): + with self.assertRaises(RuntimeError) as cm: + stt._transcribe_sync(_AUDIO, SR) + self.assertNotIsInstance(cm.exception, GpuUnavailableError) + self.assertFalse(stt._gpu_lost) + + def test_cpu_model_errors_skip_the_probe(self) -> None: + stt = _transcriber(_DeadModel()) + stt._device = "cpu" + with patch.object(Transcriber, "_cuda_alive") as probe: + with self.assertRaises(RuntimeError) as cm: + stt._transcribe_sync(_AUDIO, SR) + probe.assert_not_called() + self.assertNotIsInstance(cm.exception, GpuUnavailableError) + + +class _BrokenCudaEncoder: + def encode_batch(self, wav, lens): + raise RuntimeError("CUDA error: unknown error") + + +class _CpuEncoder: + def __init__(self) -> None: + self.calls = 0 + + def encode_batch(self, wav, lens): + self.calls += 1 + return torch.ones(1, 1, 192) + + +@unittest.skipIf(torch is None, "torch not installed") +class ProfilerCpuFallbackTests(unittest.TestCase): + def _profiler(self, base: Path) -> VoiceProfiler: + prof = VoiceProfiler(Config(paths=PathsConfig(base_dir=base))) + prof._encoder = _BrokenCudaEncoder() + prof._device = "cuda:0" + return prof + + def test_cuda_error_switches_encoder_to_cpu_once(self) -> None: + with tempfile.TemporaryDirectory() as d: + prof = self._profiler(Path(d)) + cpu = _CpuEncoder() + with patch.object(VoiceProfiler, "_build_encoder", return_value=cpu) as build: + first = prof.embed(_AUDIO, SR) + prof.embed(_AUDIO, SR) + build.assert_called_once_with("cpu") + self.assertIs(prof._encoder, cpu) + self.assertEqual(prof._device, "cpu") + self.assertEqual(cpu.calls, 2) + self.assertAlmostEqual(float(np.linalg.norm(first)), 1.0, places=5) + + def test_errors_on_cpu_are_not_swallowed(self) -> None: + with tempfile.TemporaryDirectory() as d: + prof = self._profiler(Path(d)) + prof._device = "cpu" + with patch.object(VoiceProfiler, "_build_encoder") as build: + with self.assertRaises(RuntimeError): + prof.embed(_AUDIO, SR) + build.assert_not_called() + + +class IdentifySpeakerTests(unittest.IsolatedAsyncioTestCase): + async def test_identify_failure_returns_sentinel(self) -> None: + from javis.agent.orchestrator import _ID_FAILED, Orchestrator + + orch = object.__new__(Orchestrator) + + def boom(audio, sr): + raise RuntimeError("CUDA error: unknown error") + + orch.profiler = SimpleNamespace(identify=boom) + self.assertIs(await orch._identify_speaker(_AUDIO, SR), _ID_FAILED) + orch.profiler = SimpleNamespace(identify=lambda a, sr: ("Florian", 0.8)) + self.assertEqual(await orch._identify_speaker(_AUDIO, SR), ("Florian", 0.8)) + + +if __name__ == "__main__": + unittest.main()