Handle CUDA context loss gracefully

Fall back to CPU for speaker identification and fail speech recognition with an actionable restart message when the NVIDIA driver resets.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

config.example.yaml

@@ -84,6 +84,8 @@ agent:
   max_llm_iterations: 5
   # Spoken feedback for voice users (empty string = silent).
   error_phrase: "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal."
+  # Spoken when the GPU was lost (e.g. NVIDIA driver updated while running).
+  gpu_lost_phrase: "Die Grafikkarte ist nicht mehr erreichbar, vermutlich wurde der Treiber aktualisiert. Bitte starte Javis neu."
   approval_phrase: "Soll ich das ausführen? Sag ja oder nein."
 
 memory:

src/javis/agent/orchestrator.py

@@ -18,6 +18,7 @@ from ..core.config import Config
 from ..core.events import EventBus
 from ..core.logger import get_logger
 from ..core.state import AgentState
+from ..core.hardware import GpuUnavailableError
 from ..core.tasks import cancel_and_wait, spawn
 from ..llm.conversation import Conversation
 from ..llm.hermes_client import HermesClient
@@ -54,6 +55,9 @@ log = get_logger("agent")
 SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$")
 
 # Detects whether the last sentence of a reply is a question.
+# Returned by Orchestrator._identify_speaker when speaker ID raised.
+_ID_FAILED = object()
+
 _FOLLOWUP_RE = re.compile(r'\?\s*$')
 
 
@@ -658,10 +662,16 @@ class Orchestrator:
                     raise
                 except Exception as e:  # noqa: BLE001
                     log.exception(f"turn failed: {e}")
-                    await self.bus.publish("error", {"where": "turn", "message": str(e)})
+                    gpu_lost = isinstance(e, GpuUnavailableError)
+                    await self.bus.publish(
+                        "error", {"where": "stt" if gpu_lost else "turn", "message": str(e)}
+                    )
                     await self.set_state(AgentState.ERROR)
                     # Voice users can't see the GUI error — say something.
-                    await self._speak_notice(self.config.agent.error_phrase)
+                    await self._speak_notice(
+                        self.config.agent.gpu_lost_phrase if gpu_lost
+                        else self.config.agent.error_phrase
+                    )
                     await asyncio.sleep(1.0)
                     # Recover to IDLE so the agent can accept new input after an error.
                     # Without this recovery, a single failure would leave Javis stuck
@@ -797,7 +807,7 @@ class Orchestrator:
                 if self.config.speaker_id.enabled:
                     tr, id_result = await asyncio.gather(
                         self.stt.transcribe(audio, sample_rate),
-                        asyncio.to_thread(self.profiler.identify, audio, sample_rate),
+                        self._identify_speaker(audio, sample_rate),
                     )
                 else:
                     tr = await self.stt.transcribe(audio, sample_rate)
@@ -815,7 +825,9 @@ class Orchestrator:
             _sp_name: Optional[str] = None
             _privileges: SpeakerPrivileges = SpeakerPrivileges()  # default: full access
             if self.config.speaker_id.enabled:
-                if id_result:
+                if id_result is _ID_FAILED:
+                    _privileges = self._unknown_privileges()   # fail closed, no enroll prompt
+                elif id_result:
                     sp_name, sp_conf = id_result
                     _sp_name = sp_name
                     speaker_prefix = f"[Speaker: {sp_name}] "
@@ -1243,6 +1255,14 @@ class Orchestrator:
             # Reset to IDLE so the GUI doesn't stay stuck on "listening"
             # when the turn ends without any speech.
             await self.set_state(AgentState.IDLE)
+    async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any:
+        """Run speaker ID off-loop; a failure must never abort the turn."""
+        try:
+            return await asyncio.to_thread(self.profiler.identify, audio, sample_rate)
+        except Exception as exc:  # noqa: BLE001
+            log.exception(f"[profiler] speaker identification failed: {exc}")
+            return _ID_FAILED
+
             return
 
         await self.set_state(AgentState.THINKING)
@@ -1258,7 +1278,7 @@ class Orchestrator:
             if self.config.speaker_id.enabled:
                 tr, id_result = await asyncio.gather(
                     self.stt.transcribe(audio, self.config.audio.sample_rate),
-                    asyncio.to_thread(self.profiler.identify, audio, self.config.audio.sample_rate),
+                    self._identify_speaker(audio, self.config.audio.sample_rate),
                 )
             else:
                 tr = await self.stt.transcribe(audio, self.config.audio.sample_rate)
@@ -1273,7 +1293,9 @@ class Orchestrator:
         _sp_name: Optional[str] = None
         _privileges: SpeakerPrivileges = SpeakerPrivileges()   # default: full access
         if self.config.speaker_id.enabled:
-            if id_result:
+            if id_result is _ID_FAILED:
+                _privileges = self._unknown_privileges()   # fail closed, no enroll prompt
+            elif id_result:
                 sp_name, sp_conf = id_result
                 _sp_name = sp_name
                 speaker_prefix = f"[Speaker: {sp_name}] "

src/javis/api/routes.py

@@ -16,6 +16,7 @@ import numpy as np
 from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile
 
 from ..core.config import Config
+from ..core.hardware import GpuUnavailableError
 from ..core.logger import get_logger
 from .devices import SlaveDevice, SlaveDeviceStore
 from .models import ConfigResponse, PairRequest, PairResponse, TurnResponse, WakeWordBlock, SlaveDeviceInfo
@@ -183,7 +184,11 @@ async def turn(
     except Exception as exc:  # noqa: BLE001
         # Return a spoken error instead of a bare 500 so the slave can voice it.
         log.exception(f"remote turn from {device.name!r} failed: {exc}")
-        phrase = config.agent.error_phrase
+        phrase = (
+            config.agent.gpu_lost_phrase
+            if isinstance(exc, GpuUnavailableError)
+            else config.agent.error_phrase
+        )
         err_audio, err_sr = await orch.synthesize_speech(phrase)
         return TurnResponse(
             user_text="",

src/javis/core/config.py

@@ -173,6 +173,12 @@ class AgentConfig:
     # Spoken when a voice turn fails, so users away from the screen get
     # feedback instead of silence.  Empty string disables.
     error_phrase: str = "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal."
+    # Spoken instead of error_phrase when the GPU's CUDA context died (e.g.
+    # an NVIDIA driver update while Javis was running) — needs a restart.
+    gpu_lost_phrase: str = (
+        "Die Grafikkarte ist nicht mehr erreichbar, vermutlich wurde der Treiber "
+        "aktualisiert. Bitte starte Javis neu."
+    )
     # Spoken when a tool call is waiting for approval and the window is not
     # in the foreground.  Empty string disables.
     approval_phrase: str = "Soll ich das ausführen? Sag ja oder nein."

src/javis/core/hardware.py

@@ -33,6 +33,20 @@ TORCH_INDEX_ROCM = "https://download.pytorch.org/whl/rocm6.0"
 CTRANSLATE2_REQUIRED_CUDA_DLL = "cublas64_12.dll"
 
 
+class GpuUnavailableError(RuntimeError):
+    """The CUDA context died while Javis was running (typically an NVIDIA
+    driver update or reset).  It cannot be recovered in-process — restart Javis."""
+
+
+def is_cuda_error(exc: BaseException) -> bool:
+    """True for CUDA / cuBLAS / cuDNN failures raised by torch or CTranslate2.
+
+    Out-of-memory is excluded: it is transient and the context stays usable.
+    """
+    msg = str(exc).lower()
+    return "out of memory" not in msg and any(k in msg for k in ("cuda", "cublas", "cudnn"))
+
+
 @dataclass
 class HardwareInfo:
     os_name: str = ""           # 'Windows', 'Linux', 'Darwin'

src/javis/stt/transcriber.py

@@ -12,8 +12,14 @@ from ..core.logger import get_logger
 
 log = get_logger("stt")
 
+from ..core.hardware import GpuUnavailableError, is_cuda_error
 from ..core.hardware import add_nvidia_dll_dirs as _add_nvidia_dll_dirs
 
+_GPU_LOST_MSG = (
+    "GPU (CUDA) lost — the NVIDIA driver was probably updated or reset while "
+    "Javis was running. Restart Javis to restore speech recognition."
+)
+
 # Register NVIDIA pip-package DLL dirs on PATH as early as possible so that
 # ctranslate2's internal LoadLibrary calls find cublas64_12.dll etc. before
 # the first CUDA operation is attempted.
@@ -71,6 +77,22 @@ class Transcriber:
         self._model = None
         self._batched = None
         self._force_cpu: bool = False  # set True after a CUDA runtime DLL failure
+        self._device: str = ""
+        # Set once the CUDA context died; every later call fails fast.
+        self._gpu_lost: bool = False
+
+    @staticmethod
+    def _cuda_alive() -> bool:
+        """Probe the CUDA context with a tiny host→device copy."""
+        try:
+            import ctranslate2
+            ctranslate2.StorageView.from_array(np.zeros(1, np.float32)).to_device(
+                ctranslate2.Device.cuda
+            )
+            return True
+        except Exception as exc:  # noqa: BLE001
+            log.debug(f"STT: CUDA probe failed: {exc}")
+            return "out of memory" in str(exc).lower()
 
     def _resolve_device_and_compute(self) -> tuple[str, str]:
         """Resolve the best device + compute_type for CTranslate2 / faster-whisper.
@@ -173,6 +195,7 @@ class Transcriber:
             cpu_threads=0,  # 0 = use all available cores
             download_root=str(self.config.paths.models_dir / "whisper"),
         )
+        self._device = device
         # BatchedInferencePipeline (faster-whisper >=1.1) runs transcription in
         # parallel across the batch dimension — roughly 4x faster on GPU with
         # no meaningful WER change.  CPU path skips this since batching adds
@@ -196,6 +219,8 @@ class Transcriber:
     def _transcribe_sync(self, audio: np.ndarray, sample_rate: int) -> TranscriptionResult:
         if self._model is None:
             raise RuntimeError("Transcriber not loaded — call load() first")
+        if self._gpu_lost:
+            raise GpuUnavailableError(_GPU_LOST_MSG)
         if sample_rate != 16000:
             # resample_poly is O(N) and ~100x faster than scipy.signal.resample (FFT-based).
             from fractions import Fraction
@@ -218,20 +243,30 @@ class Transcriber:
 
         def _run_transcribe() -> tuple:
             if self._batched is not None:
-                return self._batched.transcribe(
+                segments, info = self._batched.transcribe(
                     audio,
                     batch_size=self.config.stt.batch_size,
                     **transcribe_kwargs,
                 )
-            return self._model.transcribe(
+            else:
+                segments, info = self._model.transcribe(
                     audio,
                     vad_filter=False,  # VAD already applied upstream
                     **transcribe_kwargs,
                 )
+            # Segments are a lazy generator — decode here so errors raised
+            # mid-decode reach the handler below.
+            return list(segments), info
 
         try:
             segments, info = _run_transcribe()
         except RuntimeError as exc:
+            if self._device == "cuda" and is_cuda_error(exc) and not self._cuda_alive():
+                # Never free or reload the model here: destroying a CTranslate2
+                # CUDA model on a dead context aborts the whole process.
+                self._gpu_lost = True
+                log.error(f"STT: {_GPU_LOST_MSG} ({exc})")
+                raise GpuUnavailableError(_GPU_LOST_MSG) from exc
             _msg = str(exc)
             if any(k in _msg.lower() for k in ("dll", "cublas", "cannot be loaded")):
                 log.warning(
@@ -245,7 +280,6 @@ class Transcriber:
             else:
                 raise
 
-        segments = list(segments)
         text =  "".join(s.text for s in segments).strip()
         return TranscriptionResult(
             text=text,

src/javis/stt/voice_profiler.py

@@ -13,6 +13,7 @@ available so the rest of the agent continues to work unchanged.
 from __future__ import annotations
 
 import json
+import threading
 import time
 from dataclasses import dataclass, field
 from fnmatch import fnmatch as _fnmatch
@@ -22,6 +23,7 @@ from typing import Optional, Union
 import numpy as np
 
 from ..core.config import Config
+from ..core.hardware import is_cuda_error
 from ..core.logger import get_logger
 
 log = get_logger("voice_profiler")
@@ -185,6 +187,8 @@ class VoiceProfiler:
                 category=UserWarning,
             )
             # 3. speechbrain Pretrainer symlinks — harmless on Windows because
+        self._device = "cpu"          # device the encoder runs on
+        self._encoder_lock = threading.Lock()   # serialises the CPU fallback
             #    model files are copied rather than symlinked.
             warnings.filterwarnings(
                 "ignore",
@@ -192,18 +196,12 @@ class VoiceProfiler:
                 category=UserWarning,
             )
 
-            from speechbrain.inference.classifiers import EncoderClassifier
             import torch  # noqa: F401 — verify torch present
 
             # SpeechBrain's run_opts parser expects "cuda:N", not bare "cuda".
             device = "cuda:0" if self._try_cuda() else "cpu"
-            savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
+            self._encoder = self._build_encoder(device)
-            log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
+            self._device = device
-            self._encoder = EncoderClassifier.from_hparams(
-                source=_MODEL_SOURCE,
-                savedir=savedir,
-                run_opts={"device": device},
-            )
             log.info("[profiler] ECAPA-TDNN loaded")
         except Exception as exc:
             log.warning(
@@ -256,6 +254,33 @@ class VoiceProfiler:
 
     # ── embedding ─────────────────────────────────────────────────────────────
 
+    def _build_encoder(self, device: str):
+        from speechbrain.inference.classifiers import EncoderClassifier
+
+        savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
+        log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
+        return EncoderClassifier.from_hparams(
+            source=_MODEL_SOURCE,
+            savedir=savedir,
+            run_opts={"device": device},
+        )
+
+    def _fallback_to_cpu(self, failed, exc: BaseException):
+        """Swap a broken CUDA encoder for a CPU one (once, thread-safe).
+
+        CUDA errors here usually mean the context died (e.g. the NVIDIA driver
+        was updated while Javis was running), which never recovers in-process.
+        """
+        with self._encoder_lock:
+            if self._encoder is failed:
+                log.warning(
+                    f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; "
+                    "restart Javis to use the GPU again"
+                )
+                self._encoder = self._build_encoder("cpu")
+                self._device = "cpu"
+            return self._encoder
+
     def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
         """Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*."""
         if self._encoder is None:
@@ -284,8 +309,16 @@ class VoiceProfiler:
 
         wav = torch.tensor(audio_np.astype(np.float32)).unsqueeze(0)  # [1, T]
         wav_lens = torch.tensor([1.0])
+        encoder = self._encoder
+        try:
             with torch.no_grad():
-            emb = self._encoder.encode_batch(wav, wav_lens)  # [1, 1, D]
+                emb = encoder.encode_batch(wav, wav_lens)  # [1, 1, D]
+        except Exception as exc:
+            if not (self._device.startswith("cuda") and is_cuda_error(exc)):
+                raise
+            encoder = self._fallback_to_cpu(encoder, exc)
+            with torch.no_grad():
+                emb = encoder.encode_batch(wav, wav_lens)
         return _l2(emb.squeeze().cpu().numpy())
 
     # ── CRUD ──────────────────────────────────────────────────────────────────

tests/test_gpu_lost.py

@@ -0,0 +1,152 @@
+"""Surviving a dead CUDA context (e.g. NVIDIA driver updated while running)."""
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import numpy as np
+
+from javis.core.config import Config, PathsConfig
+from javis.core.hardware import GpuUnavailableError, is_cuda_error
+from javis.stt.transcriber import Transcriber
+from javis.stt.voice_profiler import VoiceProfiler
+
+try:
+    import torch
+except ImportError:  # pragma: no cover
+    torch = None
+
+SR = 16000
+_AUDIO = np.full(SR, 0.1, dtype=np.float32)
+
+
+class IsCudaErrorTests(unittest.TestCase):
+    def test_classification(self) -> None:
+        self.assertTrue(is_cuda_error(RuntimeError("CUDA error: unknown error")))
+        self.assertTrue(is_cuda_error(RuntimeError("CUDA failed with error unknown error")))
+        self.assertTrue(is_cuda_error(RuntimeError("cuBLAS failed with status X")))
+        self.assertFalse(is_cuda_error(RuntimeError("CUDA failed with error out of memory")))
+        self.assertFalse(is_cuda_error(ValueError("bad input")))
+
+
+class _DeadModel:
+    """Fake WhisperModel whose lazy segment generator hits a dead context."""
+
+    def __init__(self, msg: str = "CUDA failed with error unknown error") -> None:
+        self.msg = msg
+        self.calls = 0
+
+    def transcribe(self, audio, **kw):
+        self.calls += 1
+
+        def segments():
+            raise RuntimeError(self.msg)
+            yield  # pragma: no cover
+
+        return segments(), SimpleNamespace(language="de", duration=1.0)
+
+
+def _transcriber(model) -> Transcriber:
+    stt = Transcriber(Config())
+    stt._model = model
+    stt._batched = None
+    stt._device = "cuda"
+    return stt
+
+
+class TranscriberGpuLostTests(unittest.TestCase):
+    def test_dead_context_raises_and_then_fails_fast(self) -> None:
+        model = _DeadModel()
+        stt = _transcriber(model)
+        with patch.object(Transcriber, "_cuda_alive", return_value=False):
+            with self.assertRaisesRegex(GpuUnavailableError, "Restart Javis"):
+                stt._transcribe_sync(_AUDIO, SR)
+            with self.assertRaises(GpuUnavailableError):
+                stt._transcribe_sync(_AUDIO, SR)
+        self.assertEqual(model.calls, 1)      # second call never touched the GPU
+        self.assertIs(stt._model, model)      # freeing it would abort the process
+        self.assertFalse(stt._force_cpu)
+
+    def test_cuda_error_on_live_context_is_not_gpu_lost(self) -> None:
+        stt = _transcriber(_DeadModel("CUDA failed with error invalid argument"))
+        with patch.object(Transcriber, "_cuda_alive", return_value=True):
+            with self.assertRaises(RuntimeError) as cm:
+                stt._transcribe_sync(_AUDIO, SR)
+        self.assertNotIsInstance(cm.exception, GpuUnavailableError)
+        self.assertFalse(stt._gpu_lost)
+
+    def test_cpu_model_errors_skip_the_probe(self) -> None:
+        stt = _transcriber(_DeadModel())
+        stt._device = "cpu"
+        with patch.object(Transcriber, "_cuda_alive") as probe:
+            with self.assertRaises(RuntimeError) as cm:
+                stt._transcribe_sync(_AUDIO, SR)
+        probe.assert_not_called()
+        self.assertNotIsInstance(cm.exception, GpuUnavailableError)
+
+
+class _BrokenCudaEncoder:
+    def encode_batch(self, wav, lens):
+        raise RuntimeError("CUDA error: unknown error")
+
+
+class _CpuEncoder:
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def encode_batch(self, wav, lens):
+        self.calls += 1
+        return torch.ones(1, 1, 192)
+
+
+@unittest.skipIf(torch is None, "torch not installed")
+class ProfilerCpuFallbackTests(unittest.TestCase):
+    def _profiler(self, base: Path) -> VoiceProfiler:
+        prof = VoiceProfiler(Config(paths=PathsConfig(base_dir=base)))
+        prof._encoder = _BrokenCudaEncoder()
+        prof._device = "cuda:0"
+        return prof
+
+    def test_cuda_error_switches_encoder_to_cpu_once(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            prof = self._profiler(Path(d))
+            cpu = _CpuEncoder()
+            with patch.object(VoiceProfiler, "_build_encoder", return_value=cpu) as build:
+                first = prof.embed(_AUDIO, SR)
+                prof.embed(_AUDIO, SR)
+            build.assert_called_once_with("cpu")
+            self.assertIs(prof._encoder, cpu)
+            self.assertEqual(prof._device, "cpu")
+            self.assertEqual(cpu.calls, 2)
+            self.assertAlmostEqual(float(np.linalg.norm(first)), 1.0, places=5)
+
+    def test_errors_on_cpu_are_not_swallowed(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            prof = self._profiler(Path(d))
+            prof._device = "cpu"
+            with patch.object(VoiceProfiler, "_build_encoder") as build:
+                with self.assertRaises(RuntimeError):
+                    prof.embed(_AUDIO, SR)
+            build.assert_not_called()
+
+
+class IdentifySpeakerTests(unittest.IsolatedAsyncioTestCase):
+    async def test_identify_failure_returns_sentinel(self) -> None:
+        from javis.agent.orchestrator import _ID_FAILED, Orchestrator
+
+        orch = object.__new__(Orchestrator)
+
+        def boom(audio, sr):
+            raise RuntimeError("CUDA error: unknown error")
+
+        orch.profiler = SimpleNamespace(identify=boom)
+        self.assertIs(await orch._identify_speaker(_AUDIO, SR), _ID_FAILED)
+        orch.profiler = SimpleNamespace(identify=lambda a, sr: ("Florian", 0.8))
+        self.assertEqual(await orch._identify_speaker(_AUDIO, SR), ("Florian", 0.8))
+
+
+if __name__ == "__main__":
+    unittest.main()