This commit is contained in:
Javis committed 2026-10-10 13:20:35 +02:00
1 parent 4c16f87a3e
commit 2493c5c532
27 files changed
+8008 -568

No files matched your search

+10
View File
@@ -0,0 +1,10 @@
{
"entries": {
"spotify_pause_access_denied_unknown_speaker": {
"value": "spotify__pausePlayback mit deviceId 'a169794694f72b88e60439a45e2b506a30254979' faellt mit 'Access denied: unknown speaker' durch. Bei Spotify-Befehlern zuerst die Integration laden oder gueltige Geraete-IDs ueber getAvailableDevices ermitteln.",
"category": "spotify",
"created_at": "2026-10-10T10:43:30.486067+00:00",
"updated_at": "2026-10-10T10:43:30.486067+00:00"
}
}
}
+10
View File
@@ -0,0 +1,10 @@
{
"entries": {
"user_name": {
"value": "Der Nutzer heisst Florian.",
"category": "personal",
"created_at": "2026-10-10T10:40:14.388969+00:00",
"updated_at": "2026-10-10T10:40:14.388969+00:00"
}
}
}
+19 -1
View File
@@ -1,4 +1,22 @@
{ {
"version": 1, "version": 1,
"commands": {} "commands": {
"abfbc899-8ad1-436b-b386-152c80669dcd": {
"id": "abfbc899-8ad1-436b-b386-152c80669dcd",
"trigger": "küche ein",
"match_mode": "contains",
"actions": [
{
"tool": "homey__control_zone",
"args": {
"capability": "onoff",
"value": true,
"zone": "Küche"
}
}
],
"created_at": "2026-10-10T10:39:38.382223+00:00",
"updated_at": "2026-10-10T10:39:38.382223+00:00"
}
}
} }
+13
View File
@@ -0,0 +1,13 @@
{
"devices": {
"a32f0630-04e5-4eda-9ae2-44fa4eeb6068": {
"name": "Tablet K\u00fcche",
"api_key_hash": "89db0c92e376e0160a756f8285d940ba8813d2867cd699badac4deb2b85b813b",
"room": "K\u00fcche",
"ip_address": "10.10.10.225",
"created_at": "2026-10-07T17:40:24+00:00",
"last_seen": "2026-10-10T10:44:31+00:00",
"last_seen_ip": "10.10.10.225"
}
}
}
Binary file not shown.
+9
View File
@@ -0,0 +1,9 @@
[
{
"name": "Florian",
"file": "Florian.npy",
"enrolled_at": "2026-10-07T21:32:39",
"clip_count": 13,
"last_seen": "2026-10-10T12:44:06"
}
]
@@ -0,0 +1 @@
[]
+27
View File
@@ -89,6 +89,33 @@ Push-to-talk: `Ctrl+Space` in the window, `Ctrl+Alt+Space` system-wide (Windows)
3. Javis will VAD-split, transcribe, and build a clean dataset. 3. Javis will VAD-split, transcribe, and build a clean dataset.
4. Test with zero-shot cloning (instant) or click **Fine-tune** for higher quality. 4. Test with zero-shot cloning (instant) or click **Fine-tune** for higher quality.
Fine-tuning runs in a console window of its own. Meanwhile Javis speaks with the
voice being trained, refreshed with the training progress every 15 minutes. Stop it
any time — Ctrl+C, closing the window, logging off or shutting down saves the
epoch so far, and the next run continues with its next batch. A crash loses at most
the last 10 minutes (`JAVIS_PIPER_PROGRESS_MINUTES`); training restarts by itself.
## Speaker identification
With `speaker_id.enabled`, Javis recognises who is talking (ECAPA embeddings) and manages
speakers in **Intelligence → Voice Profiles**:
- **Enroll** — `+` records a few sentences and creates a profile.
- **Unidentified speakers** — utterances that matched no profile are kept (audio,
transcript, time, source device, closest profile). **▶ Play** listens to one;
**Sort…** adds it to an existing profile or creates a new one. Recordings that sound
alike are offered and pre-selected so you can sort several at once. ✕ / **Clear all**
delete them.
- **Enhance a voice** — 🎤 on a profile card opens a recorder: click **● Record sample**,
speak, review each sample (playback + how well it matches the profile; mismatches are
flagged), then **Add N samples** to blend them in.
- The "unknown speaker" pop-up (`speaker_id.auto_enroll`) can also play back what was said.
Unknown-speaker clips are stored locally only, in `<voice_profiles_dir>/unidentified/`.
They are capped by `speaker_id.max_unidentified` (default 50, oldest dropped first). Turn
the inbox off with `speaker_id.keep_unidentified: false`. Both settings are under
**Settings → Speaker Identification**.
## MCP servers ## MCP servers
Edit `mcp_servers.json`. Schema matches VS Code's `mcp.json`, so you can copy/paste server entries. Edit `mcp_servers.json`. Schema matches VS Code's `mcp.json`, so you can copy/paste server entries.
+1 -1
View File
@@ -40,7 +40,7 @@ training:
batch_size: 32 batch_size: 32
num_workers: 2 num_workers: 2
accelerator: "gpu" # gpu | cpu accelerator: "gpu" # gpu | cpu
precision: "16-mixed" # 32 | 16-mixed | bf16-mixed precision: "bf16-mixed" # bf16-mixed (fastest) | 32 | 16-mixed
language: "de" # espeak-ng voice code used for phonemisation (de, en-us, …) language: "de" # espeak-ng voice code used for phonemisation (de, en-us, …)
audio: audio:
+6 -3
View File
@@ -494,9 +494,12 @@ The overnight scheduler runs `run_pipeline()` via `asyncio.to_thread`; the threa
on the training subprocess for hours. Cancelling the scheduler task (quit) released on the training subprocess for hours. Cancelling the scheduler task (quit) released
the training lock immediately while the thread kept training, and at interpreter exit the training lock immediately while the thread kept training, and at interpreter exit
Python joins executor threads — so Javis hung on quit until training finished. Python joins executor threads — so Javis hung on quit until training finished.
**Fix:** cancellation calls `piper_train.request_stop()`, which terminates the child **Fix:** cancellation calls `piper_train.request_stop()`: the training process finishes
process (`_run` now polls instead of blocking in `wait()`); the lock is only released its step, saves its progress — within the epoch too: the next run continues with the
once the worker has actually finished. next batch — and exits (it is killed if that takes longer than
`STOP_GRACE_SECONDS`), and the scheduler waits for that (`_run` polls instead of
blocking in `wait()`). The lock (`training/.training.lock`, held by `run_pipeline`)
names the training processes and is only released once they have exited.
#### VOI-01 #### VOI-01
**Imported voice archives may carry pickle checkpoints** — *Medium (trust)* **Imported voice archives may carry pickle checkpoints** — *Medium (trust)*
+4
View File
@@ -37,6 +37,10 @@ screen (GUI mode) and played back through the device's speakers.
If the master's turn fails it returns its spoken error phrase instead, so a If the master's turn fails it returns its spoken error phrase instead, so a
screenless satellite still gets audible feedback; connection problems play screenless satellite still gets audible feedback; connection problems play
a low falling tone. a low falling tone.
5. **Conversation view** — The GUI lists the conversation with the newest
message on top; scroll down for older ones. The fullscreen button in the
conversation card (or `F11`) toggles window fullscreen; the orb and the
conversation stay visible. `Esc` leaves fullscreen.
## Setup ## Setup
+78 -18
View File
@@ -132,6 +132,7 @@ QPushButton[variant="danger"] {{ color: {CL_ERROR_FG}; }}
QPushButton[variant="danger"]:hover {{ background: {CL_ERROR_BG}; border-color: {CL_ERROR_BORDER}; }} QPushButton[variant="danger"]:hover {{ background: {CL_ERROR_BG}; border-color: {CL_ERROR_BORDER}; }}
QPushButton[variant="icon"] {{ padding: 0; min-height: 0; border-radius: 16px; }} QPushButton[variant="icon"] {{ padding: 0; min-height: 0; border-radius: 16px; }}
QPushButton[variant="icon"]:checked {{ background: {CL_ERROR}; border-color: {CL_ERROR}; }} QPushButton[variant="icon"]:checked {{ background: {CL_ERROR}; border-color: {CL_ERROR}; }}
QPushButton[variant="icon"][small="true"] {{ min-width: 46px; min-height: 46px; }}
QScrollArea {{ background: transparent; border: 0; }} QScrollArea {{ background: transparent; border: 0; }}
QScrollBar:vertical {{ background: transparent; width: 8px; margin: 4px 0; }} QScrollBar:vertical {{ background: transparent; width: 8px; margin: 4px 0; }}
@@ -211,9 +212,28 @@ def _draw_tune(p: QtGui.QPainter, box: QtCore.QRectF, color: QtGui.QColor) -> No
p.drawEllipse(QtCore.QPointF(x + knob * s, y + row * s), 0.09 * s, 0.09 * s) p.drawEllipse(QtCore.QPointF(x + knob * s, y + row * s), 0.09 * s, 0.09 * s)
def _draw_expand(p: QtGui.QPainter, box: QtCore.QRectF, color: QtGui.QColor, collapse: bool) -> None:
"""Four corner brackets: pointing outwards to enter, inwards to leave fullscreen."""
s, x, y = box.width(), box.x(), box.y()
pen = QtGui.QPen(color, max(1.5, s * 0.075))
pen.setCapStyle(QtCore.Qt.RoundCap)
pen.setJoinStyle(QtCore.Qt.RoundJoin)
p.setPen(pen)
p.setBrush(QtCore.Qt.NoBrush)
leg = 0.26 * s
reach = 0.10 if collapse else 0.36 # corner distance from the centre
d = 1 if collapse else -1 # legs run away from / towards the centre
for sx in (-1, 1):
for sy in (-1, 1):
vx, vy = x + (0.5 + sx * reach) * s, y + (0.5 + sy * reach) * s
p.drawPolyline([QtCore.QPointF(vx + d * sx * leg, vy),
QtCore.QPointF(vx, vy),
QtCore.QPointF(vx, vy + d * sy * leg)])
@lru_cache(maxsize=None) @lru_cache(maxsize=None)
def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon: def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon:
"""Render a crisp (2x) icon: 'mic', 'mic_off' or 'tune'.""" """Render a crisp (2x) icon: 'mic', 'mic_off', 'tune', 'expand' or 'collapse'."""
pm = QtGui.QPixmap(size * 2, size * 2) pm = QtGui.QPixmap(size * 2, size * 2)
pm.setDevicePixelRatio(2.0) pm.setDevicePixelRatio(2.0)
pm.fill(QtCore.Qt.transparent) pm.fill(QtCore.Qt.transparent)
@@ -222,6 +242,8 @@ def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon:
box = QtCore.QRectF(0, 0, size, size) box = QtCore.QRectF(0, 0, size, size)
if kind == "tune": if kind == "tune":
_draw_tune(p, box, QtGui.QColor(color)) _draw_tune(p, box, QtGui.QColor(color))
elif kind in ("expand", "collapse"):
_draw_expand(p, box, QtGui.QColor(color), collapse=(kind == "collapse"))
else: else:
_draw_mic(p, box, QtGui.QColor(color), slashed=(kind == "mic_off")) _draw_mic(p, box, QtGui.QColor(color), slashed=(kind == "mic_off"))
p.end() p.end()
@@ -574,15 +596,18 @@ class MainScreen(QtWidgets.QWidget):
Alexa-style UX features: Alexa-style UX features:
- Mute toggle button on the orb - Mute toggle button on the orb
- Connection health monitoring with retry countdown - Connection health monitoring with retry countdown
- Transcript history (scrollable conversation log) - Transcript history (scrollable conversation log, newest message on top)
- Fullscreen button in the conversation card (orb and chat stay visible)
- Visual mute indicator overlay - Visual mute indicator overlay
""" """
settingsRequested = QtCore.Signal() settingsRequested = QtCore.Signal()
muteToggled = QtCore.Signal(bool) # new signal for mute toggle muteToggled = QtCore.Signal(bool) # new signal for mute toggle
fullscreenToggleRequested = QtCore.Signal()
def __init__(self, parent=None): def __init__(self, parent=None):
super().__init__(parent) super().__init__(parent)
self._fullscreen = False
self.setFocusPolicy(QtCore.Qt.ClickFocus) # tapping the background clears focus self.setFocusPolicy(QtCore.Qt.ClickFocus) # tapping the background clears focus
outer = QtWidgets.QVBoxLayout(self) outer = QtWidgets.QVBoxLayout(self)
outer.setContentsMargins(28, 20, 28, 24) outer.setContentsMargins(28, 20, 28, 24)
@@ -654,10 +679,24 @@ class MainScreen(QtWidgets.QWidget):
cv.setContentsMargins(24, 18, 14, 18) cv.setContentsMargins(24, 18, 14, 18)
cv.setSpacing(10) cv.setSpacing(10)
# Header label # Header row: caption + fullscreen-chat toggle
header_row = QtWidgets.QHBoxLayout()
header_row.setContentsMargins(0, 0, 0, 0)
history_header = QtWidgets.QLabel("Konversation") history_header = QtWidgets.QLabel("Konversation")
history_header.setProperty("role", "caption") history_header.setProperty("role", "caption")
cv.addWidget(history_header) header_row.addWidget(history_header)
header_row.addStretch(1)
self.expand_btn = QtWidgets.QPushButton()
self.expand_btn.setProperty("variant", "icon")
self.expand_btn.setProperty("small", True) # 48 px: the stylesheet's min-height would shrink it
self.expand_btn.setFixedSize(48, 48)
self.expand_btn.setIconSize(QtCore.QSize(26, 26))
self.expand_btn.setCursor(QtCore.Qt.PointingHandCursor)
self.expand_btn.setFocusPolicy(QtCore.Qt.TabFocus)
self.expand_btn.clicked.connect(self.fullscreenToggleRequested.emit)
self._sync_expand_button()
header_row.addWidget(self.expand_btn)
cv.addLayout(header_row)
# Transcript history (replaces single-turn display). # Transcript history (replaces single-turn display).
# NOTE: this layout must belong to *exactly one* parent — we set it on # NOTE: this layout must belong to *exactly one* parent — we set it on
@@ -708,6 +747,20 @@ class MainScreen(QtWidgets.QWidget):
self.mute_btn.setToolTip(tip) self.mute_btn.setToolTip(tip)
self.mute_btn.setAccessibleName(tip) self.mute_btn.setAccessibleName(tip)
def set_fullscreen(self, fullscreen: bool) -> None:
"""Mirror the window's fullscreen state in the toggle button."""
if fullscreen == self._fullscreen:
return
self._fullscreen = fullscreen
self._sync_expand_button()
def _sync_expand_button(self) -> None:
fs = self._fullscreen
self.expand_btn.setIcon(_icon("collapse" if fs else "expand", CL_TEXT, 26))
tip = "Vollbild beenden" if fs else "Vollbild"
self.expand_btn.setToolTip(tip)
self.expand_btn.setAccessibleName(tip)
def _bubble_max_width(self) -> int: def _bubble_max_width(self) -> int:
return max(240, int(self.transcript_scroll.viewport().width() * 0.8)) return max(240, int(self.transcript_scroll.viewport().width() * 0.8))
@@ -767,7 +820,8 @@ class MainScreen(QtWidgets.QWidget):
else: else:
h.addWidget(bubble) h.addWidget(bubble)
h.addStretch(1) h.addStretch(1)
self.transcript_layout.addWidget(row) # Newest first: the latest bubble goes above the older ones.
self.transcript_layout.insertWidget(0, row)
# ── slots (called via Qt signals from the engine bridge) ────────────── # ── slots (called via Qt signals from the engine bridge) ──────────────
@@ -796,9 +850,8 @@ class MainScreen(QtWidgets.QWidget):
def _ensure_transcript_spacer(self) -> None: def _ensure_transcript_spacer(self) -> None:
"""Keep exactly one expanding spacer, at the end of the transcript. """Keep exactly one expanding spacer, at the end of the transcript.
Bubbles are appended after the previous spacer, so stale spacers must It pushes short transcripts to the top; stray spacers (e.g. left from
be removed from anywhere in the layout (they used to pile up between the empty state) would open gaps between bubbles.
turns and open large gaps).
""" """
for i in reversed(range(self.transcript_layout.count())): for i in reversed(range(self.transcript_layout.count())):
item = self.transcript_layout.itemAt(i) item = self.transcript_layout.itemAt(i)
@@ -809,16 +862,16 @@ class MainScreen(QtWidgets.QWidget):
self.transcript_layout.addSpacerItem(spacer) self.transcript_layout.addSpacerItem(spacer)
def _prune_transcript(self, max_bubbles: int = 40) -> None: def _prune_transcript(self, max_bubbles: int = 40) -> None:
"""Keep at most *max_bubbles* bubbles in the transcript layout (oldest first).""" """Keep at most *max_bubbles* bubbles in the transcript layout (drops the oldest, at the bottom)."""
rows = [self.transcript_layout.itemAt(i).widget() rows = [self.transcript_layout.itemAt(i).widget()
for i in range(self.transcript_layout.count())] for i in range(self.transcript_layout.count())]
rows = [row for row in rows if row is not None] rows = [row for row in rows if row is not None]
for row in rows[:max(0, len(rows) - max_bubbles)]: for row in rows[max_bubbles:]:
self.transcript_layout.removeWidget(row) self.transcript_layout.removeWidget(row)
row.deleteLater() row.deleteLater()
def set_turn(self, user_text: str, reply_text: str) -> None: def set_turn(self, user_text: str, reply_text: str) -> None:
"""Add a new conversation turn to the transcript history (Alexa-style).""" """Add a new conversation turn to the transcript history (newest on top)."""
self._add_bubble(user_text, "user") self._add_bubble(user_text, "user")
# Assistant message bubble (if any) # Assistant message bubble (if any)
@@ -826,16 +879,16 @@ class MainScreen(QtWidgets.QWidget):
self._add_bubble(reply_text, "assistant") self._add_bubble(reply_text, "assistant")
self._prune_transcript() self._prune_transcript()
# Spacer to push content to bottom (like Alexa's conversation history) # Keeps short transcripts hugging the top edge
self._ensure_transcript_spacer() self._ensure_transcript_spacer()
# Auto-scroll to bottom # Auto-scroll to top
QTimer.singleShot(50, self._scroll_transcript_to_bottom) QTimer.singleShot(50, self._scroll_transcript_to_top)
def _scroll_transcript_to_bottom(self) -> None: def _scroll_transcript_to_top(self) -> None:
"""Auto-scroll transcript to show latest messages.""" """Auto-scroll transcript to show the latest messages (newest is on top)."""
vbar = self.transcript_scroll.verticalScrollBar() vbar = self.transcript_scroll.verticalScrollBar()
vbar.setValue(vbar.maximum()) vbar.setValue(vbar.minimum())
def set_connection(self, ok: bool, retry_count: int = 0, backoff: float = 0.0) -> None: def set_connection(self, ok: bool, retry_count: int = 0, backoff: float = 0.0) -> None:
if ok: if ok:
@@ -865,7 +918,7 @@ class MainScreen(QtWidgets.QWidget):
self._add_bubble(f"⚠ {message}", "error") self._add_bubble(f"⚠ {message}", "error")
self._prune_transcript() self._prune_transcript()
self._ensure_transcript_spacer() self._ensure_transcript_spacer()
QTimer.singleShot(50, self._scroll_transcript_to_bottom) QTimer.singleShot(50, self._scroll_transcript_to_top)
class SlaveWindow(QtWidgets.QMainWindow): class SlaveWindow(QtWidgets.QMainWindow):
@@ -907,6 +960,7 @@ class SlaveWindow(QtWidgets.QMainWindow):
self.engineError.connect(self.main_screen.show_error) self.engineError.connect(self.main_screen.show_error)
self.engineConnection.connect(self.main_screen.set_connection) self.engineConnection.connect(self.main_screen.set_connection)
self.engineRoom.connect(lambda r: self.main_screen.set_room(r)) self.engineRoom.connect(lambda r: self.main_screen.set_room(r))
self.main_screen.fullscreenToggleRequested.connect(self._toggle_fullscreen)
QtGui.QShortcut(QtGui.QKeySequence("F11"), self, activated=self._toggle_fullscreen) QtGui.QShortcut(QtGui.QKeySequence("F11"), self, activated=self._toggle_fullscreen)
# Kiosk: fullscreen with a hidden cursor (touch panels); Esc leaves fullscreen. # Kiosk: fullscreen with a hidden cursor (touch panels); Esc leaves fullscreen.
@@ -919,6 +973,12 @@ class SlaveWindow(QtWidgets.QMainWindow):
self.showFullScreen() self.showFullScreen()
QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CursorShape.BlankCursor) QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CursorShape.BlankCursor)
def changeEvent(self, event): # noqa: N802
super().changeEvent(event)
if event.type() == QtCore.QEvent.Type.WindowStateChange:
# Covers every path in (button, F11, kiosk, settings) and out (Esc).
self.main_screen.set_fullscreen(self.isFullScreen())
def _leave_fullscreen(self) -> None: def _leave_fullscreen(self) -> None:
if self.isFullScreen(): if self.isFullScreen():
self.showNormal() self.showNormal()
+358 -41
View File
@@ -2,9 +2,11 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import contextlib
import json import json
import re import re
import time import time
from collections.abc import AsyncIterator
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Optional from typing import Any, Optional
@@ -16,9 +18,9 @@ from ..audio_io.vad import make_vad
from ..audio_io.wake_word import WakeWordDetector from ..audio_io.wake_word import WakeWordDetector
from ..core.config import Config from ..core.config import Config
from ..core.events import EventBus from ..core.events import EventBus
from ..core.hardware import GpuUnavailableError
from ..core.logger import get_logger from ..core.logger import get_logger
from ..core.state import AgentState from ..core.state import AgentState
from ..core.hardware import GpuUnavailableError
from ..core.tasks import cancel_and_wait, spawn from ..core.tasks import cancel_and_wait, spawn
from ..llm.conversation import Conversation from ..llm.conversation import Conversation
from ..llm.hermes_client import HermesClient from ..llm.hermes_client import HermesClient
@@ -48,16 +50,17 @@ from .middleware import (
drain_background_tasks, drain_background_tasks,
) )
from .prompts import load_system_prompt from .prompts import load_system_prompt
from ..stt.unidentified import UnidentifiedStore
from ..stt.voice_profiler import NullProfiler, SpeakerPrivileges, VoiceProfiler from ..stt.voice_profiler import NullProfiler, SpeakerPrivileges, VoiceProfiler
log = get_logger("agent") log = get_logger("agent")
SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$")
# Detects whether the last sentence of a reply is a question.
# Returned by Orchestrator._identify_speaker when speaker ID raised. # Returned by Orchestrator._identify_speaker when speaker ID raised.
_ID_FAILED = object() _ID_FAILED = object()
SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$")
# Detects whether the last sentence of a reply is a question.
_FOLLOWUP_RE = re.compile(r'\?\s*$') _FOLLOWUP_RE = re.compile(r'\?\s*$')
@@ -353,6 +356,37 @@ class RemoteTurnResult:
expects_followup: bool = False expects_followup: bool = False
@dataclass
class VoiceSample:
"""A live voice sample recorded from the GUI to enhance a profile."""
audio: np.ndarray
sample_rate: int
embedding: Optional[np.ndarray] = None
# Cosine similarity to the target profile (None if it does not exist yet).
score: Optional[float] = None
best_name: Optional[str] = None
best_score: Optional[float] = None
@property
def duration_s(self) -> float:
return len(self.audio) / float(self.sample_rate) if self.sample_rate else 0.0
@dataclass
class _PendingEnrollClip:
"""Unknown-speaker clip buffered while the "who is this?" prompt is open."""
audio: np.ndarray
sample_rate: int
# Task storing the same clip in the unidentified inbox (returns its id).
store_task: Optional["asyncio.Task[Optional[str]]"] = None
def inbox_id(self) -> Optional[str]:
t = self.store_task
if t is None or not t.done() or t.cancelled() or t.exception() is not None:
return None
return t.result()
class Orchestrator: class Orchestrator:
"""Long-lived voice agent. Owns all subsystems.""" """Long-lived voice agent. Owns all subsystems."""
@@ -441,8 +475,16 @@ class Orchestrator:
self.profiler: VoiceProfiler | NullProfiler = ( self.profiler: VoiceProfiler | NullProfiler = (
VoiceProfiler(config) if config.speaker_id.enabled else NullProfiler() VoiceProfiler(config) if config.speaker_id.enabled else NullProfiler()
) )
self._enroll_clip_buffer: list[np.ndarray] = [] self.unidentified = UnidentifiedStore(
config.paths.voice_profiles_dir / "unidentified",
config.speaker_id.max_unidentified,
)
self._enroll_clip_buffer: list[_PendingEnrollClip] = []
self._enroll_ask_pending: bool = False self._enroll_ask_pending: bool = False
# Cleared while the GUI records voice samples so the wake-word loop
# (which reads the mic outside _turn_lock) stops competing for chunks.
self._mic_free = asyncio.Event()
self._mic_free.set()
self._bg_quick_turn_count: int = 0 self._bg_quick_turn_count: int = 0
self._tool_cache: dict[str, tuple[str, float]] = {} self._tool_cache: dict[str, tuple[str, float]] = {}
@@ -560,8 +602,13 @@ class Orchestrator:
# Disable any backend whose load() raised so later synthesis/prewarm # Disable any backend whose load() raised so later synthesis/prewarm
# calls don't encounter RuntimeError("load() has not been called"). # calls don't encounter RuntimeError("load() has not been called").
if self.tts is not None and getattr(self.tts, "_voice", True) is None: if self.tts is not None and getattr(self.tts, "_voice", True) is None:
log.warning("TTS model failed to load — speech output disabled") backend, self.tts = self.tts, None
self.tts = None if hasattr(backend, "model_available"):
log.warning("TTS model failed to load — speech output disabled until "
"a model is available")
spawn(self._revive_tts(backend), name="tts-revive")
else:
log.warning("TTS model failed to load — speech output disabled")
await self.bus.publish("mcp_ready", self.mcp.status_snapshot()) await self.bus.publish("mcp_ready", self.mcp.status_snapshot())
ok = await self.llm.health() ok = await self.llm.health()
await self.bus.publish("llm_health", {"ok": ok, "url": self.config.llm.base_url, "model": self.config.llm.model}) await self.bus.publish("llm_health", {"ok": ok, "url": self.config.llm.base_url, "model": self.config.llm.model})
@@ -854,8 +901,12 @@ class Orchestrator:
await self.bus.publish( await self.bus.publish(
"speaker_id", {"name": None, "confidence": 0.0, "known": False} "speaker_id", {"name": None, "confidence": 0.0, "known": False}
) )
if self.config.speaker_id.auto_enroll: source = device_name or "satellite"
await self._auto_enroll_speaker(audio) if room:
source = f"{source} ({room})"
await self._handle_unknown_speaker(
audio, sample_rate, text, source=source,
)
if not _privileges.can_respond: if not _privileges.can_respond:
log.info( log.info(
@@ -947,6 +998,10 @@ class Orchestrator:
False when the listen loop exited without a detection (stale timeout, False when the listen loop exited without a detection (stale timeout,
``_stop`` set, or mic disconnected).""" ``_stop`` set, or mic disconnected)."""
assert self.wake is not None assert self.wake is not None
# A GUI voice-sample recording owns the mic — wait until it is done.
mic_free: Optional[asyncio.Event] = getattr(self, "_mic_free", None)
if mic_free is not None and not mic_free.is_set():
await mic_free.wait()
await self.set_state(AgentState.IDLE) await self.set_state(AgentState.IDLE)
self.wake.reset() self.wake.reset()
if self.mic is not None: if self.mic is not None:
@@ -954,6 +1009,8 @@ class Orchestrator:
log.info(f"listening for wake word: '{self.config.wake_word.model}'") log.info(f"listening for wake word: '{self.config.wake_word.model}'")
last_pub = 0.0 last_pub = 0.0
async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S): async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S):
if mic_free is not None and not mic_free.is_set():
return False # handed over to an exclusive recording
score = self.wake.feed(chunk) score = self.wake.feed(chunk)
# Everything heard before the wake word is background (music, TV, # Everything heard before the wake word is background (music, TV,
# this device's own speakers), so the recorder's voice gate is # this device's own speakers), so the recorder's voice gate is
@@ -1213,6 +1270,14 @@ class Orchestrator:
return True return True
return None return None
async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any:
"""Run speaker ID off-loop; a failure must never abort the turn."""
try:
return await asyncio.to_thread(self.profiler.identify, audio, sample_rate)
except Exception as exc: # noqa: BLE001
log.exception(f"[profiler] speaker identification failed: {exc}")
return _ID_FAILED
def _unknown_privileges(self) -> SpeakerPrivileges: def _unknown_privileges(self) -> SpeakerPrivileges:
"""Build a :class:`SpeakerPrivileges` from the unknown-speaker config fields.""" """Build a :class:`SpeakerPrivileges` from the unknown-speaker config fields."""
cfg = self.config.speaker_id cfg = self.config.speaker_id
@@ -1228,6 +1293,32 @@ class Orchestrator:
denied_tools=denied, denied_tools=denied,
) )
async def _revive_tts(self, backend, interval: float = 30.0) -> None:
"""Turn speech output on once the TTS model that failed to load is there.
E.g. a voice still in its first training run: training exports the
voice's model as it goes (see ``javis.voices.piper_train``).
"""
while True:
try:
await asyncio.wait_for(self._stop.wait(), timeout=interval)
return
except asyncio.TimeoutError:
pass
try:
if not backend.model_available():
continue
await asyncio.to_thread(backend.load)
except Exception as exc: # noqa: BLE001 - tried again once the model changes
log.warning(f"TTS model still failed to load: {exc}")
continue
if self._stop.is_set():
return
self.tts = backend
log.info("TTS model loaded — speech output enabled")
spawn(self._prewarm_tts(), name="tts-prewarm")
return
async def _prewarm_tts(self) -> None: async def _prewarm_tts(self) -> None:
"""Silently synthesize a short phrase to warm up TTS model buffers. """Silently synthesize a short phrase to warm up TTS model buffers.
@@ -1270,14 +1361,6 @@ class Orchestrator:
# Reset to IDLE so the GUI doesn't stay stuck on "listening" # Reset to IDLE so the GUI doesn't stay stuck on "listening"
# when the turn ends without any speech. # when the turn ends without any speech.
await self.set_state(AgentState.IDLE) await self.set_state(AgentState.IDLE)
async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any:
"""Run speaker ID off-loop; a failure must never abort the turn."""
try:
return await asyncio.to_thread(self.profiler.identify, audio, sample_rate)
except Exception as exc: # noqa: BLE001
log.exception(f"[profiler] speaker identification failed: {exc}")
return _ID_FAILED
return return
await self.set_state(AgentState.THINKING) await self.set_state(AgentState.THINKING)
@@ -1347,8 +1430,9 @@ class Orchestrator:
await self.bus.publish( await self.bus.publish(
"speaker_id", {"name": None, "confidence": 0.0, "known": False} "speaker_id", {"name": None, "confidence": 0.0, "known": False}
) )
if self.config.speaker_id.auto_enroll: await self._handle_unknown_speaker(
await self._auto_enroll_speaker(audio) audio, self.config.audio.sample_rate, tr.text or "", source="local",
)
# ── privilege: can_respond check ────────────────────────────────────── # ── privilege: can_respond check ──────────────────────────────────────
if not _privileges.can_respond: if not _privileges.can_respond:
@@ -2007,8 +2091,7 @@ class Orchestrator:
n = self.config.speaker_id.min_enroll_clips n = self.config.speaker_id.min_enroll_clips
clips: list[np.ndarray] = [] clips: list[np.ndarray] = []
async with self._turn_lock: async with self._exclusive_mic():
await self.set_state(AgentState.ENROLLING)
for i in range(n): for i in range(n):
await self.bus.publish("enroll_progress", { await self.bus.publish("enroll_progress", {
"name": name, "clip": i, "total": n, "recording": True, "name": name, "clip": i, "total": n, "recording": True,
@@ -2016,7 +2099,6 @@ class Orchestrator:
spawn(self._play_wake_chime()) spawn(self._play_wake_chime())
audio = await record_utterance(self.config, self.mic, self.vad) audio = await record_utterance(self.config, self.mic, self.vad)
if audio is None or len(audio) < 1600: if audio is None or len(audio) < 1600:
await self.set_state(AgentState.IDLE)
await self.bus.publish("enroll_progress", { await self.bus.publish("enroll_progress", {
"name": name, "clip": i, "total": n, "name": name, "clip": i, "total": n,
"error": "Kein Audio erkannt — Aufnahme abgebrochen.", "error": "Kein Audio erkannt — Aufnahme abgebrochen.",
@@ -2030,48 +2112,283 @@ class Orchestrator:
sr = self.config.audio.sample_rate sr = self.config.audio.sample_rate
await asyncio.to_thread(self.profiler.enroll, name, clips, sr) await asyncio.to_thread(self.profiler.enroll, name, clips, sr)
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)}) await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)})
await self.set_state(AgentState.IDLE)
log.info(f"[profiler] enrollment complete for '{name}'") log.info(f"[profiler] enrollment complete for '{name}'")
return True return True
@contextlib.asynccontextmanager
async def _exclusive_mic(self, wait_s: float = 0.0) -> AsyncIterator[MicStream]:
"""Own the microphone for a GUI-initiated recording.
Takes ``_turn_lock`` (waiting up to *wait_s*) and pauses the wake-word
loop via ``_mic_free`` so every chunk reaches the recorder. Raises
``RuntimeError`` with a user-facing message when that is not possible.
"""
mic = self.mic
if mic is None:
raise RuntimeError("No microphone available — voice input is disabled.")
try:
await asyncio.wait_for(self._turn_lock.acquire(), timeout=max(0.01, wait_s))
except asyncio.TimeoutError:
raise RuntimeError("Javis is busy right now — try again in a moment.") from None
self._mic_free.clear()
try:
await self.set_state(AgentState.ENROLLING)
mic.drain()
yield mic
finally:
self._mic_free.set()
try:
await self.set_state(AgentState.IDLE)
finally:
self._turn_lock.release()
def _require_profiler(self) -> None:
if not self.config.speaker_id.enabled:
raise RuntimeError(
"Speaker identification is disabled (Settings → Speaker Identification).")
if not self.profiler.is_ready():
raise RuntimeError("The speaker identification model is not loaded yet.")
async def record_voice_sample(
self, name: Optional[str] = None, *, max_seconds: float = 15.0,
) -> VoiceSample:
"""Record one live utterance for enhancing *name*'s voice profile.
Nothing is stored — pass the result to :meth:`add_voice_samples` once
the user has reviewed it. ``score`` is the cosine similarity to
*name*'s current profile (``None`` if it has none). Raises
``RuntimeError`` with a user-facing message on failure.
"""
self._require_profiler()
sr = self.config.audio.sample_rate
async with self._exclusive_mic(wait_s=3.0) as mic:
# Chime first, then drop whatever the mic picked up of it.
await self._play_wake_chime()
mic.drain()
audio = await record_utterance(
self.config, mic, self.vad,
max_seconds=max_seconds, initial_grace_ms=5000,
)
if audio is None or len(audio) < int(sr * 0.5):
raise RuntimeError("No speech detected — please try again.")
emb = await asyncio.to_thread(self.profiler.embed, audio, sr)
ranked = self.profiler.rank(emb)
score = next((s for n, s in ranked if n == name), None) if name else None
best_name, best_score = ranked[0] if ranked else (None, None)
return VoiceSample(
audio=audio, sample_rate=sr, embedding=emb, score=score,
best_name=best_name, best_score=best_score,
)
async def add_voice_samples(self, name: str, samples: list[VoiceSample]) -> int:
"""Blend recorded *samples* into *name*'s profile (created if missing).
Returns the number of samples used.
"""
name = (name or "").strip()
if not name:
raise ValueError("Profile name is empty.")
def _apply() -> tuple[int, bool]:
embs = []
for s in samples:
emb = s.embedding
if emb is None or not np.any(emb):
emb = self.profiler.embed(s.audio, s.sample_rate)
if np.any(emb):
embs.append(emb)
if not embs:
return 0, False
if self.profiler.has_profile(name):
used = sum(1 for e in embs if self.profiler.refine_embedding(name, e))
return used, False
return (len(embs), True) if self.profiler.enroll_embeddings(name, embs) else (0, False)
used, created = await asyncio.to_thread(_apply)
if not used:
raise RuntimeError("No usable voice samples — is the speaker model loaded?")
log.info(f"[profiler] added {used} live sample(s) to '{name}'")
await self.bus.publish("voice_profiles_changed", {
"name": name, "added": used, "created": created,
})
return used
# ---------------- unidentified speakers ----------------
async def _handle_unknown_speaker(
self, audio: np.ndarray, sample_rate: int, text: str, *, source: str,
) -> None:
"""Keep the clip in the unidentified inbox and/or ask for a name."""
store_task: Optional["asyncio.Task[Optional[str]]"] = None
if self.config.speaker_id.keep_unidentified and (text or "").strip():
store_task = spawn(
self._store_unidentified(audio, sample_rate, text, source),
name="unidentified-store",
)
if self.config.speaker_id.auto_enroll:
await self._auto_enroll_speaker(audio, sample_rate, store_task)
async def _store_unidentified(
self, audio: np.ndarray, sample_rate: int, text: str, source: str,
) -> Optional[str]:
"""Embed *audio* and add it to the inbox. Returns the clip id."""
try:
emb = await asyncio.to_thread(self.profiler.embed, audio, sample_rate)
ranked = self.profiler.rank(emb)
best_name, best_score = ranked[0] if ranked else (None, None)
self.unidentified.max_clips = max(1, int(self.config.speaker_id.max_unidentified))
clip_id = await asyncio.to_thread(
self.unidentified.add, audio, sample_rate,
embedding=emb, text=text, source=source,
best_name=best_name, best_score=best_score,
)
except Exception as exc: # noqa: BLE001
log.warning(f"[unidentified] could not store clip: {exc}")
return None
await self.bus.publish("unidentified_changed", {"added": clip_id})
return clip_id
async def assign_unidentified(self, clip_ids: list[str], name: str) -> dict:
"""Sort inbox clips into *name*'s profile, creating it if needed.
Returns ``{"name", "added", "created"}``. The clips are removed from
the inbox. Raises ``ValueError``/``RuntimeError`` with a user-facing
message on failure.
"""
name = (name or "").strip()
if not name:
raise ValueError("Profile name is empty.")
if not clip_ids:
raise ValueError("No clips selected.")
self._require_profiler()
def _apply() -> tuple[list[str], bool]:
used: list[str] = []
embs: list[np.ndarray] = []
for cid in clip_ids:
emb = self.unidentified.load_embedding(cid)
if emb is None:
try:
audio, sr = self.unidentified.load_audio(cid)
except Exception as exc: # noqa: BLE001
log.warning(f"[unidentified] clip {cid} unavailable: {exc}")
continue
emb = self.profiler.embed(audio, sr)
if np.any(emb):
embs.append(emb)
used.append(cid)
if not embs:
return [], False
if self.profiler.has_profile(name):
for e in embs:
self.profiler.refine_embedding(name, e)
created = False
else:
if not self.profiler.enroll_embeddings(name, embs):
return [], False
created = True
self.unidentified.remove(used)
return used, created
used, created = await asyncio.to_thread(_apply)
if not used:
raise RuntimeError("None of the selected clips could be used.")
# Don't let a still-open "who is this?" prompt enroll them a second time.
done = set(used)
self._enroll_clip_buffer = [
p for p in self._enroll_clip_buffer if p.inbox_id() not in done
]
log.info(
f"[profiler] sorted {len(used)} unidentified clip(s) into "
f"{'new' if created else 'existing'} profile '{name}'"
)
await self.bus.publish("voice_profiles_changed", {
"name": name, "added": len(used), "created": created,
})
await self.bus.publish("unidentified_changed", {"removed": used})
return {"name": name, "added": len(used), "created": created}
async def delete_unidentified(self, clip_ids: Optional[list[str]] = None) -> int:
"""Delete the given inbox clips, or all of them when *clip_ids* is None."""
if clip_ids is None:
removed = await asyncio.to_thread(self.unidentified.clear)
else:
removed = await asyncio.to_thread(self.unidentified.remove, clip_ids)
if removed:
await self.bus.publish("unidentified_changed", {"removed": clip_ids})
return removed
def pending_enroll_clips(self) -> list[tuple[np.ndarray, int]]:
"""Clips buffered for the open "who is this?" prompt (for playback)."""
return [(p.audio, p.sample_rate) for p in self._enroll_clip_buffer]
def auto_enroll_dismiss(self) -> None:
"""The user skipped the name prompt.
Drops the buffered clips (they stay in the unidentified inbox when
``keep_unidentified`` is on) and lets the prompt appear again later.
"""
self._enroll_clip_buffer.clear()
self._enroll_ask_pending = False
async def auto_enroll_confirm(self, key: str, name: str) -> bool: async def auto_enroll_confirm(self, key: str, name: str) -> bool:
"""Finalize auto-enrollment using all clips buffered since the dialog opened. """Finalize auto-enrollment using all clips buffered since the dialog opened.
Called by the GUI when the user submits their name. *key* is kept for Called by the GUI when the user submits their name. *key* is kept for
API compatibility but is no longer used to look up audio — the full API compatibility but is no longer used to look up audio — the full
clip buffer is consumed instead. clip buffer is consumed instead. Clips that were also stored in the
unidentified inbox are removed from it.
""" """
clips = list(self._enroll_clip_buffer) pending = list(self._enroll_clip_buffer)
self._enroll_clip_buffer.clear() self._enroll_clip_buffer.clear()
self._enroll_ask_pending = False self._enroll_ask_pending = False
if not clips: if not pending:
log.warning("[profiler] auto_enroll_confirm: clip buffer is empty") log.warning("[profiler] auto_enroll_confirm: clip buffer is empty")
return False return False
sr = self.config.audio.sample_rate def _apply() -> int:
existing_names = {p["name"] for p in self.profiler.list_profiles()} embs = [self.profiler.embed(p.audio, p.sample_rate) for p in pending]
if name in existing_names: if self.profiler.has_profile(name):
# Merge all buffered clips into the existing profile. # Merge all buffered clips into the existing profile.
for clip in clips: n = sum(1 for e in embs if self.profiler.refine_embedding(name, e))
await asyncio.to_thread(self.profiler.refine, name, clip, sr) log.info(f"[profiler] merged {n} clip(s) into existing profile '{name}'")
log.info( return n
f"[profiler] merged {len(clips)} clip(s) into existing profile '{name}'"
)
else:
# Build a new profile from all accumulated clips. # Build a new profile from all accumulated clips.
await asyncio.to_thread(self.profiler.enroll, name, clips, sr) return len(embs) if self.profiler.enroll_embeddings(name, embs) else 0
log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)")
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)}) used = await asyncio.to_thread(_apply)
if not used:
return False
inbox_ids: list[str] = []
for p in pending:
if p.store_task is not None:
try:
cid = await p.store_task
except Exception: # noqa: BLE001
cid = None
if cid:
inbox_ids.append(cid)
if inbox_ids:
await self.delete_unidentified(inbox_ids)
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": used})
return True return True
async def _auto_enroll_speaker(self, audio: np.ndarray) -> None: async def _auto_enroll_speaker(
self,
audio: np.ndarray,
sample_rate: Optional[int] = None,
store_task: Optional["asyncio.Task[Optional[str]]"] = None,
) -> None:
"""Buffer *audio* from an unknown speaker and ask for a name (once). """Buffer *audio* from an unknown speaker and ask for a name (once).
Subsequent unknown utterances while the dialog is open are silently Subsequent unknown utterances while the dialog is open are silently
appended to the buffer so enrollment ultimately uses multiple clips. appended to the buffer so enrollment ultimately uses multiple clips.
""" """
self._enroll_clip_buffer.append(audio) self._enroll_clip_buffer.append(_PendingEnrollClip(
audio, int(sample_rate or self.config.audio.sample_rate), store_task,
))
if not self._enroll_ask_pending: if not self._enroll_ask_pending:
self._enroll_ask_pending = True self._enroll_ask_pending = True
# Use a stable key so the GUI can match confirm callbacks. # Use a stable key so the GUI can match confirm callbacks.
+12 -3
View File
@@ -253,6 +253,11 @@ class SpeakerIDConfig:
# auto-refinement. Only high-confidence identifications (score >= # auto-refinement. Only high-confidence identifications (score >=
# threshold + refine_margin) update the profile, keeping noise clips out. # threshold + refine_margin) update the profile, keeping noise clips out.
refine_margin: float = 0.10 refine_margin: float = 0.10
# Keep audio clips of unidentified speakers (WAV + embedding, stored under
# voice_profiles_dir/unidentified) so they can be played back and sorted
# into a profile from the GUI. Capped at max_unidentified (oldest dropped).
keep_unidentified: bool = True
max_unidentified: int = 50
# ── Unknown / guest speaker privileges ─────────────────────────────────── # ── Unknown / guest speaker privileges ───────────────────────────────────
# Applied when the speaker cannot be matched to any known voice profile. # Applied when the speaker cannot be matched to any known voice profile.
# Controlled via the GUI "Settings → Speaker Identification" section. # Controlled via the GUI "Settings → Speaker Identification" section.
@@ -295,7 +300,9 @@ class TrainingConfig:
batch_size: int = 32 batch_size: int = 32
num_workers: int = 2 num_workers: int = 2
accelerator: str = "gpu" # gpu | cpu accelerator: str = "gpu" # gpu | cpu
precision: str = "16-mixed" # 32 | 16-mixed | bf16-mixed # bf16-mixed trains fastest and needs no loss scaling; GPUs without bf16
# fall back to 32 at launch.
precision: str = "bf16-mixed" # 32 | 16-mixed | bf16-mixed
language: str = "de" # espeak-ng voice code (de, en-us, ...) language: str = "de" # espeak-ng voice code (de, en-us, ...)
@@ -430,6 +437,8 @@ def validate(cfg: "Config") -> None:
cfg.speaker_id.threshold, 0.0, 1.0, 0.65, "speaker_id.threshold") cfg.speaker_id.threshold, 0.0, 1.0, 0.65, "speaker_id.threshold")
cfg.speaker_id.refine_margin = _clamp( cfg.speaker_id.refine_margin = _clamp(
cfg.speaker_id.refine_margin, 0.0, 0.5, 0.10, "speaker_id.refine_margin") cfg.speaker_id.refine_margin, 0.0, 0.5, 0.10, "speaker_id.refine_margin")
cfg.speaker_id.max_unidentified = _clamp_int(
cfg.speaker_id.max_unidentified, 1, 500, 50, "speaker_id.max_unidentified")
cfg.wake_word.vad_gate = _clamp( cfg.wake_word.vad_gate = _clamp(
cfg.wake_word.vad_gate, 0.0, 1.0, 0.0, "wake_word.vad_gate") cfg.wake_word.vad_gate, 0.0, 1.0, 0.0, "wake_word.vad_gate")
cfg.wake_word.voice_gate_ratio = _clamp( cfg.wake_word.voice_gate_ratio = _clamp(
@@ -512,8 +521,8 @@ def validate(cfg: "Config") -> None:
if t.precision not in {"32", "16-mixed", "bf16-mixed"}: if t.precision not in {"32", "16-mixed", "bf16-mixed"}:
log.warning( log.warning(
f"config: training.precision='{t.precision}' is unknown " f"config: training.precision='{t.precision}' is unknown "
"(32|16-mixed|bf16-mixed), using '16-mixed'") "(32|16-mixed|bf16-mixed), using 'bf16-mixed'")
t.precision = "16-mixed" t.precision = "bf16-mixed"
if cfg.memory.provider not in {"ollama", "hermes"}: if cfg.memory.provider not in {"ollama", "hermes"}:
log.warning( log.warning(
File diff suppressed because it is too large. Load diff
+186
View File
@@ -0,0 +1,186 @@
"""Inbox of voice clips from speakers that speaker ID could not match.
Each clip is kept so the user can listen to it in the GUI and sort it into an
existing voice profile or use it to create a new one::
~/.javis/voice_profiles/unidentified/{id}.wav 16-bit PCM audio
~/.javis/voice_profiles/unidentified/{id}.npy ECAPA embedding (optional)
~/.javis/voice_profiles/unidentified/index.json metadata, oldest first
The inbox is capped at ``max_clips``; the oldest clips are dropped first.
All methods are thread-safe (clips are added from worker threads).
"""
from __future__ import annotations
import json
import threading
import time
import uuid
from collections.abc import Iterable
from pathlib import Path
import numpy as np
from ..core.logger import get_logger
log = get_logger("unidentified")
_INDEX_FILE = "index.json"
def _mono(audio: np.ndarray) -> np.ndarray:
a = np.asarray(audio, dtype=np.float32)
if a.ndim > 1:
a = a.mean(axis=1)
return a.reshape(-1)
class UnidentifiedStore:
"""Bounded on-disk store of unidentified speaker clips."""
def __init__(self, directory: Path, max_clips: int = 50) -> None:
self.dir = Path(directory)
self.max_clips = max(1, int(max_clips))
self._lock = threading.RLock()
self._index: list[dict] | None = None # lazily loaded
# ── index I/O ─────────────────────────────────────────────────────────────
def _entries(self) -> list[dict]:
if self._index is None:
idx = self.dir / _INDEX_FILE
data: list = []
if idx.exists():
try:
data = json.loads(idx.read_text(encoding="utf-8"))
except Exception as exc: # noqa: BLE001
log.warning(f"[unidentified] index read failed: {exc}")
self._index = [
e for e in data
if isinstance(e, dict) and e.get("id") and e.get("file")
and (self.dir / e["file"]).exists()
]
return self._index
def _save(self) -> None:
self.dir.mkdir(parents=True, exist_ok=True)
idx = self.dir / _INDEX_FILE
tmp = idx.with_suffix(".json.part")
try:
tmp.write_text(
json.dumps(self._entries(), ensure_ascii=False, indent=2),
encoding="utf-8",
)
tmp.replace(idx)
except Exception as exc: # noqa: BLE001
log.error(f"[unidentified] index write failed: {exc}")
def _delete_files(self, entry: dict) -> None:
for key in ("file", "embedding"):
fname = entry.get(key)
if not fname:
continue
try:
(self.dir / fname).unlink(missing_ok=True)
except Exception as exc: # noqa: BLE001
log.warning(f"[unidentified] could not delete {fname}: {exc}")
# ── public API ────────────────────────────────────────────────────────────
def add(
self,
audio: np.ndarray,
sample_rate: int,
*,
embedding: np.ndarray | None = None,
text: str = "",
source: str = "",
best_name: str | None = None,
best_score: float | None = None,
) -> str:
"""Store a clip and return its id. Prunes the oldest beyond the cap."""
import soundfile as sf
mono = np.clip(_mono(audio), -1.0, 1.0)
clip_id = f"{time.strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:6]}"
with self._lock:
self.dir.mkdir(parents=True, exist_ok=True)
wav_name = f"{clip_id}.wav"
sf.write(str(self.dir / wav_name), mono, int(sample_rate), subtype="PCM_16")
emb_name = None
if embedding is not None and np.any(embedding):
emb_name = f"{clip_id}.npy"
np.save(str(self.dir / emb_name),
np.asarray(embedding, dtype=np.float32).reshape(-1))
entries = self._entries()
entries.append({
"id": clip_id,
"file": wav_name,
"embedding": emb_name,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
"duration_s": round(len(mono) / float(sample_rate), 2),
"sample_rate": int(sample_rate),
"text": (text or "").strip(),
"source": source or "",
"best_name": best_name,
"best_score": None if best_score is None else round(float(best_score), 4),
})
while len(entries) > self.max_clips:
self._delete_files(entries.pop(0))
self._save()
log.info(f"[unidentified] stored clip {clip_id} ({len(mono) / sample_rate:.1f}s)")
return clip_id
def entries(self) -> list[dict]:
"""All clip metadata, newest first (copies — safe to keep)."""
with self._lock:
return [dict(e) for e in reversed(self._entries())]
def get(self, clip_id: str) -> dict | None:
with self._lock:
entry = next((e for e in self._entries() if e["id"] == clip_id), None)
return dict(entry) if entry else None
def load_audio(self, clip_id: str) -> tuple[np.ndarray, int]:
"""Return ``(float32 mono audio, sample_rate)``; raises ``KeyError`` if unknown."""
import soundfile as sf
entry = self.get(clip_id)
if entry is None:
raise KeyError(clip_id)
audio, sr = sf.read(str(self.dir / entry["file"]), dtype="float32", always_2d=False)
return _mono(audio), int(sr)
def load_embedding(self, clip_id: str) -> np.ndarray | None:
entry = self.get(clip_id)
if entry is None or not entry.get("embedding"):
return None
try:
return np.load(str(self.dir / entry["embedding"])).astype(np.float32)
except Exception as exc: # noqa: BLE001
log.warning(f"[unidentified] embedding for {clip_id} unreadable: {exc}")
return None
def remove(self, clip_ids: Iterable[str]) -> int:
"""Delete the given clips; returns how many were removed."""
wanted = set(clip_ids)
if not wanted:
return 0
with self._lock:
entries = self._entries()
gone = [e for e in entries if e["id"] in wanted]
if not gone:
return 0
for e in gone:
self._delete_files(e)
self._index = [e for e in entries if e["id"] not in wanted]
self._save()
return len(gone)
def clear(self) -> int:
with self._lock:
return self.remove([e["id"] for e in self._entries()])
def __len__(self) -> int:
with self._lock:
return len(self._entries())
+180 -88
View File
@@ -47,6 +47,16 @@ def _l2(v: np.ndarray) -> np.ndarray:
return v / n if n > 0 else v return v / n if n > 0 else v
def _as_vector(embedding) -> Optional[np.ndarray]:
"""Flatten *embedding* to float32; ``None`` for missing or all-zero vectors."""
if embedding is None:
return None
vec = np.asarray(embedding, dtype=np.float32).reshape(-1)
if vec.size == 0 or not np.any(vec):
return None
return vec
# Minimum utterance duration for a reliable embedding. Shorter clips are # Minimum utterance duration for a reliable embedding. Shorter clips are
# passed through but logged as a warning — ECAPA was trained on ≥ 1 s segments. # passed through but logged as a warning — ECAPA was trained on ≥ 1 s segments.
_MIN_EMBED_DURATION_S = 0.5 _MIN_EMBED_DURATION_S = 0.5
@@ -125,6 +135,28 @@ class NullProfiler:
def identify(self, audio_np: np.ndarray, sr: int) -> Optional[tuple[str, float]]: # noqa: ARG002 def identify(self, audio_np: np.ndarray, sr: int) -> Optional[tuple[str, float]]: # noqa: ARG002
return None return None
def identify_embedding(self, embedding) -> Optional[tuple[str, float]]: # noqa: ARG002
return None
def is_ready(self) -> bool:
return False
def has_profile(self, name: str) -> bool: # noqa: ARG002
return False
def rank(self, embedding) -> list[tuple[str, float]]: # noqa: ARG002
return []
def enroll_embeddings(self, name: str, embeddings: list) -> bool: # noqa: ARG002
log.warning("[profiler] NullProfiler — speechbrain unavailable, enroll skipped")
return False
def refine(self, name: str, clip: np.ndarray, sr: int) -> None: # noqa: ARG002
pass
def refine_embedding(self, name: str, embedding) -> bool: # noqa: ARG002
return False
def list_profiles(self) -> list[dict]: def list_profiles(self) -> list[dict]:
return [] return []
@@ -155,8 +187,13 @@ class VoiceProfiler:
self.config = config self.config = config
self._profiles_dir: Path = config.paths.voice_profiles_dir self._profiles_dir: Path = config.paths.voice_profiles_dir
self._encoder = None # speechbrain EncoderClassifier or None self._encoder = None # speechbrain EncoderClassifier or None
self._device = "cpu" # device the encoder runs on
self._encoder_lock = threading.Lock() # serialises the CPU fallback
self._profiles: dict[str, np.ndarray] = {} # name → L2-normed embedding self._profiles: dict[str, np.ndarray] = {} # name → L2-normed embedding
self._index: list[dict] = [] # metadata records self._index: list[dict] = [] # metadata records
# identify/refine run on worker threads while the GUI enrolls/deletes
# on the loop thread — serialise access to _profiles/_index.
self._lock = threading.RLock()
# ── lifecycle ───────────────────────────────────────────────────────────── # ── lifecycle ─────────────────────────────────────────────────────────────
@@ -164,9 +201,18 @@ class VoiceProfiler:
"""Load the ECAPA-TDNN encoder and all enrolled profiles from disk.""" """Load the ECAPA-TDNN encoder and all enrolled profiles from disk."""
self._profiles_dir.mkdir(parents=True, exist_ok=True) self._profiles_dir.mkdir(parents=True, exist_ok=True)
self._load_encoder() self._load_encoder()
self._load_index() with self._lock:
self._load_index()
log.info(f"[profiler] ready — {len(self._profiles)} profile(s) loaded") log.info(f"[profiler] ready — {len(self._profiles)} profile(s) loaded")
def is_ready(self) -> bool:
"""``True`` once the ECAPA encoder is loaded (embeddings are meaningful)."""
return self._encoder is not None
def has_profile(self, name: str) -> bool:
with self._lock:
return name in self._profiles
def _load_encoder(self) -> None: def _load_encoder(self) -> None:
try: try:
import os import os
@@ -187,8 +233,6 @@ class VoiceProfiler:
category=UserWarning, category=UserWarning,
) )
# 3. speechbrain Pretrainer symlinks — harmless on Windows because # 3. speechbrain Pretrainer symlinks — harmless on Windows because
self._device = "cpu" # device the encoder runs on
self._encoder_lock = threading.Lock() # serialises the CPU fallback
# model files are copied rather than symlinked. # model files are copied rather than symlinked.
warnings.filterwarnings( warnings.filterwarnings(
"ignore", "ignore",
@@ -210,6 +254,33 @@ class VoiceProfiler:
) )
self._encoder = None self._encoder = None
def _build_encoder(self, device: str):
from speechbrain.inference.classifiers import EncoderClassifier
savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
return EncoderClassifier.from_hparams(
source=_MODEL_SOURCE,
savedir=savedir,
run_opts={"device": device},
)
def _fallback_to_cpu(self, failed, exc: BaseException):
"""Swap a broken CUDA encoder for a CPU one (once, thread-safe).
CUDA errors here usually mean the context died (e.g. the NVIDIA driver
was updated while Javis was running), which never recovers in-process.
"""
with self._encoder_lock:
if self._encoder is failed:
log.warning(
f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; "
"restart Javis to use the GPU again"
)
self._encoder = self._build_encoder("cpu")
self._device = "cpu"
return self._encoder
@staticmethod @staticmethod
def _try_cuda() -> bool: def _try_cuda() -> bool:
try: try:
@@ -254,33 +325,6 @@ class VoiceProfiler:
# ── embedding ───────────────────────────────────────────────────────────── # ── embedding ─────────────────────────────────────────────────────────────
def _build_encoder(self, device: str):
from speechbrain.inference.classifiers import EncoderClassifier
savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
return EncoderClassifier.from_hparams(
source=_MODEL_SOURCE,
savedir=savedir,
run_opts={"device": device},
)
def _fallback_to_cpu(self, failed, exc: BaseException):
"""Swap a broken CUDA encoder for a CPU one (once, thread-safe).
CUDA errors here usually mean the context died (e.g. the NVIDIA driver
was updated while Javis was running), which never recovers in-process.
"""
with self._encoder_lock:
if self._encoder is failed:
log.warning(
f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; "
"restart Javis to use the GPU again"
)
self._encoder = self._build_encoder("cpu")
self._device = "cpu"
return self._encoder
def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray: def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
"""Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*.""" """Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*."""
if self._encoder is None: if self._encoder is None:
@@ -328,25 +372,58 @@ class VoiceProfiler:
if self._encoder is None: if self._encoder is None:
log.warning("[profiler] enroll skipped — encoder not loaded") log.warning("[profiler] enroll skipped — encoder not loaded")
return return
vecs = [self.embed(c, sr) for c in clips] self.enroll_embeddings(name, [self.embed(c, sr) for c in clips])
avg = _l2(np.mean(vecs, axis=0))
# Build a filesystem-safe filename from the profile name. def enroll_embeddings(self, name: str, embeddings: list[np.ndarray]) -> bool:
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in name) """Create (or replace) *name*'s profile from precomputed embeddings.
npy_path = self._profiles_dir / f"{safe}.npy"
np.save(str(npy_path), avg)
self._profiles[name] = avg
# Replace existing entry if the name already exists. All-zero vectors (from an unloaded encoder) are ignored. Returns
self._index = [e for e in self._index if e["name"] != name] ``False`` when no usable embedding remains.
self._index.append({ """
"name": name, vecs = [v for v in (_as_vector(e) for e in embeddings) if v is not None]
"file": f"{safe}.npy", if not vecs:
"enrolled_at": time.strftime("%Y-%m-%dT%H:%M:%S"), log.warning(f"[profiler] enroll '{name}' skipped — no usable embeddings")
"clip_count": len(clips), return False
}) avg = _l2(np.mean(vecs, axis=0)).astype(np.float32)
self._save_index()
log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)") with self._lock:
self._profiles_dir.mkdir(parents=True, exist_ok=True)
old = next((e for e in self._index if e["name"] == name), None)
if old is not None:
fname = old["file"]
else:
# Filesystem-safe filename; never reuse another profile's file
# ("Anna B" and "Anna_B" sanitise to the same stem).
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in name)
used = {e["file"] for e in self._index}
fname, i = f"{safe}.npy", 2
while fname in used:
fname, i = f"{safe}_{i}.npy", i + 1
np.save(str(self._profiles_dir / fname), avg)
self._profiles[name] = avg
# Replace existing entry if the name already exists.
self._index = [e for e in self._index if e["name"] != name]
self._index.append({
"name": name,
"file": fname,
"enrolled_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
"clip_count": len(vecs),
})
self._save_index()
log.info(f"[profiler] enrolled '{name}' from {len(vecs)} clip(s)")
return True
def rank(self, embedding) -> list[tuple[str, float]]:
"""Cosine similarity of *embedding* to every profile, best first."""
vec = _as_vector(embedding)
if vec is None:
return []
with self._lock:
items = list(self._profiles.items())
scores = [(n, _cosine(vec, e)) for n, e in items if e.shape == vec.shape]
scores.sort(key=lambda x: -x[1])
return scores
def identify( def identify(
self, audio_np: np.ndarray, sr: int self, audio_np: np.ndarray, sr: int
@@ -358,20 +435,19 @@ class VoiceProfiler:
""" """
if self._encoder is None or not self._profiles: if self._encoder is None or not self._profiles:
return None return None
query = self.embed(audio_np, sr) return self.identify_embedding(self.embed(audio_np, sr))
threshold = self.config.speaker_id.threshold
scores: dict[str, float] = { def identify_embedding(self, embedding) -> Optional[tuple[str, float]]:
name: _cosine(query, emb) for name, emb in self._profiles.items() """Like :meth:`identify` for an embedding computed by :meth:`embed`."""
} ranked = self.rank(embedding)
best_name = max(scores, key=scores.__getitem__) if not ranked:
best_score = scores[best_name] return None
threshold = self.config.speaker_id.threshold
best_name, best_score = ranked[0]
# Always log all candidate scores so identification failures are visible # Always log all candidate scores so identification failures are visible
# in normal (INFO-level) logs without needing to enable DEBUG mode. # in normal (INFO-level) logs without needing to enable DEBUG mode.
score_str = ", ".join( score_str = ", ".join(f"{n}={s:.3f}" for n, s in ranked)
f"{n}={s:.3f}" for n, s in sorted(scores.items(), key=lambda x: -x[1])
)
if best_score >= threshold: if best_score >= threshold:
log.info(f"[profiler] identified '{best_name}' [{score_str}] threshold={threshold}") log.info(f"[profiler] identified '{best_name}' [{score_str}] threshold={threshold}")
return best_name, float(best_score) return best_name, float(best_score)
@@ -382,7 +458,8 @@ class VoiceProfiler:
def list_profiles(self) -> list[dict]: def list_profiles(self) -> list[dict]:
"""Return a list of profile metadata dicts from the index.""" """Return a list of profile metadata dicts from the index."""
return list(self._index) with self._lock:
return list(self._index)
def refine(self, name: str, clip: np.ndarray, sr: int) -> None: def refine(self, name: str, clip: np.ndarray, sr: int) -> None:
"""Blend a new voice sample into *name*'s existing profile. """Blend a new voice sample into *name*'s existing profile.
@@ -393,36 +470,50 @@ class VoiceProfiler:
""" """
if self._encoder is None: if self._encoder is None:
return return
entry = next((e for e in self._index if e["name"] == name), None) if not self.has_profile(name):
if entry is None or name not in self._profiles:
log.warning(f"[profiler] refine: no profile found for '{name}'") log.warning(f"[profiler] refine: no profile found for '{name}'")
return return
new_emb = self.embed(clip, sr) self.refine_embedding(name, self.embed(clip, sr))
n = entry.get("clip_count", 1)
current = self._profiles[name] def refine_embedding(self, name: str, embedding) -> bool:
# Running centroid: (current * n + new) / (n+1), then L2-normalise. """Blend a precomputed embedding into *name*'s centroid (see :meth:`refine`)."""
blended = _l2(current * n + new_emb) new_emb = _as_vector(embedding)
npy_path = self._profiles_dir / entry["file"] if new_emb is None:
np.save(str(npy_path), blended) return False
self._profiles[name] = blended with self._lock:
entry["clip_count"] = n + 1 entry = next((e for e in self._index if e["name"] == name), None)
entry["last_seen"] = time.strftime("%Y-%m-%dT%H:%M:%S") if entry is None or name not in self._profiles:
self._save_index() log.warning(f"[profiler] refine: no profile found for '{name}'")
return False
n = entry.get("clip_count", 1)
current = self._profiles[name]
if current.shape != new_emb.shape:
log.warning(f"[profiler] refine '{name}': embedding shape mismatch")
return False
# Running centroid: (current * n + new) / (n+1), then L2-normalise.
blended = _l2(current * n + new_emb).astype(np.float32)
np.save(str(self._profiles_dir / entry["file"]), blended)
self._profiles[name] = blended
entry["clip_count"] = n + 1
entry["last_seen"] = time.strftime("%Y-%m-%dT%H:%M:%S")
self._save_index()
log.debug(f"[profiler] refined '{name}' — centroid now over {n + 1} clip(s)") log.debug(f"[profiler] refined '{name}' — centroid now over {n + 1} clip(s)")
return True
def delete_profile(self, name: str) -> bool: def delete_profile(self, name: str) -> bool:
"""Remove *name*'s embedding from disk and memory.""" """Remove *name*'s embedding from disk and memory."""
entry = next((e for e in self._index if e["name"] == name), None) with self._lock:
if entry is None: entry = next((e for e in self._index if e["name"] == name), None)
return False if entry is None:
npy = self._profiles_dir / entry["file"] return False
try: npy = self._profiles_dir / entry["file"]
npy.unlink(missing_ok=True) try:
except Exception as exc: npy.unlink(missing_ok=True)
log.warning(f"[profiler] delete {npy.name} failed: {exc}") except Exception as exc:
self._index = [e for e in self._index if e["name"] != name] log.warning(f"[profiler] delete {npy.name} failed: {exc}")
self._profiles.pop(name, None) self._index = [e for e in self._index if e["name"] != name]
self._save_index() self._profiles.pop(name, None)
self._save_index()
log.info(f"[profiler] deleted profile '{name}'") log.info(f"[profiler] deleted profile '{name}'")
return True return True
@@ -437,10 +528,11 @@ class VoiceProfiler:
def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None: def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None:
"""Persist updated *privileges* for *name* to ``index.json``.""" """Persist updated *privileges* for *name* to ``index.json``."""
entry = next((e for e in self._index if e["name"] == name), None) with self._lock:
if entry is None: entry = next((e for e in self._index if e["name"] == name), None)
log.warning(f"[profiler] set_privileges: no profile found for '{name}'") if entry is None:
return log.warning(f"[profiler] set_privileges: no profile found for '{name}'")
entry["privileges"] = privileges.to_dict() return
self._save_index() entry["privileges"] = privileges.to_dict()
self._save_index()
log.info(f"[profiler] privileges updated for '{name}'") log.info(f"[profiler] privileges updated for '{name}'")
+129 -34
View File
@@ -12,6 +12,11 @@ The model path is resolved in this order:
1. ``config.tts.piper_model`` (explicit path in config.yaml) 1. ``config.tts.piper_model`` (explicit path in config.yaml)
2. ``{voices_dir}/{tts.default_voice}/piper_model.onnx`` (active voice dir) 2. ``{voices_dir}/{tts.default_voice}/piper_model.onnx`` (active voice dir)
The model is reloaded by itself when that changes: when training updates the
voice's model (it does every few minutes, see ``javis.voices.piper_train``) or
another voice is activated. Until the new model is loaded, the previous one
keeps speaking.
Inference is synchronous (ONNX runtime) so it runs in a thread-pool executor Inference is synchronous (ONNX runtime) so it runs in a thread-pool executor
to avoid blocking the asyncio event loop. to avoid blocking the asyncio event loop.
@@ -23,6 +28,7 @@ from __future__ import annotations
import asyncio import asyncio
import re import re
import threading import threading
import time
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
@@ -59,42 +65,42 @@ def _clean(text: str) -> str:
class PiperBackend(TTSBackend): class PiperBackend(TTSBackend):
"""Piper ONNX TTS backend.""" """Piper ONNX TTS backend."""
# Synthesis looks for a changed model at most this often.
CHECK_SECONDS = 2.0
def __init__(self, config: Config) -> None: def __init__(self, config: Config) -> None:
self.config = config self.config = config
self._voice: Optional[object] = None # piper.voice.PiperVoice self._voice: Optional[object] = None # piper.voice.PiperVoice
self._sample_rate: int = 22050 self._sample_rate: int = 22050
self._lock = threading.Lock() self._lock = threading.Lock() # one synthesis at a time; guards the swap
self._reloading = threading.Lock()
self._loaded: tuple | None = None # _files_signature() of the loaded model
self._failed: tuple | None = None # ... of a model that failed to load
self._checked = 0.0
# ── TTSBackend interface ────────────────────────────────────────────────── # ── TTSBackend interface ──────────────────────────────────────────────────
def load(self) -> None: def load(self) -> None:
"""Load the ONNX model. Called once at startup.""" """Load the ONNX model. Called once at startup."""
try:
from piper.voice import PiperVoice # type: ignore[import]
except ImportError as exc:
raise RuntimeError(
"piper-tts is not installed. Run: pip install piper-tts"
) from exc
model_path = self._resolve_model_path() model_path = self._resolve_model_path()
config_path = Path(str(model_path) + ".json") signature = _files_signature(model_path)
try:
log.info(f"Loading Piper model: {model_path}") voice = self._load_voice(model_path)
self._voice = PiperVoice.load( except BaseException:
str(model_path), self._failed = signature
config_path=str(config_path) if config_path.exists() else None, raise
use_cuda=False, # ONNX runtime picks CUDA automatically when available with self._lock:
) self._install(voice, signature)
self._sample_rate = (
self._voice.config.sample_rate # type: ignore[union-attr]
if hasattr(self._voice, "config")
else 22050
)
log.info( log.info(
f"Piper model loaded — sample_rate={self._sample_rate} Hz, " f"Piper model loaded — sample_rate={self._sample_rate} Hz, "
f"language={getattr(getattr(self._voice, 'config', None), 'espeak_voice', '?')}" f"language={getattr(getattr(voice, 'config', None), 'espeak_voice', '?')}"
) )
def model_available(self) -> bool:
"""Whether there is a model for :meth:`load` (other than one it failed to load)."""
model_path = self._model_path()
return model_path is not None and _files_signature(model_path) != self._failed
async def synthesize( async def synthesize(
self, self,
text: str, text: str,
@@ -108,13 +114,17 @@ class PiperBackend(TTSBackend):
sample_rate=self._sample_rate, sample_rate=self._sample_rate,
) )
audio = await asyncio.to_thread(self._synthesize_sync, text) audio, sample_rate = await asyncio.to_thread(self._synthesize_sync, text)
return SynthesisResult(audio=audio, sample_rate=self._sample_rate) return SynthesisResult(audio=audio, sample_rate=sample_rate)
# ── Internal ────────────────────────────────────────────────────────────── # ── Internal ──────────────────────────────────────────────────────────────
def _synthesize_sync(self, text: str) -> np.ndarray: def _synthesize_sync(self, text: str) -> tuple[np.ndarray, int]:
"""Run Piper inference synchronously (called from thread-pool).""" """Run Piper inference synchronously (called from thread-pool).
Returns the audio and its sample rate.
"""
self._check_for_new_model()
if self._voice is None: if self._voice is None:
raise RuntimeError("PiperBackend.load() has not been called.") raise RuntimeError("PiperBackend.load() has not been called.")
@@ -128,31 +138,103 @@ class PiperBackend(TTSBackend):
chunks: list[np.ndarray] = [] chunks: list[np.ndarray] = []
with self._lock: with self._lock:
sample_rate = self._sample_rate
for chunk in self._voice.synthesize(text, syn_config): for chunk in self._voice.synthesize(text, syn_config):
arr = chunk.audio_float_array arr = chunk.audio_float_array
if arr is not None and len(arr) > 0: if arr is not None and len(arr) > 0:
chunks.append(np.asarray(arr, dtype=np.float32)) chunks.append(np.asarray(arr, dtype=np.float32))
if not chunks: if not chunks:
return np.zeros(0, dtype=np.float32) return np.zeros(0, dtype=np.float32), sample_rate
return np.concatenate(chunks) return np.concatenate(chunks), sample_rate
def _resolve_model_path(self) -> Path: @staticmethod
"""Return the ONNX model path from config or voice profile dir.""" def _load_voice(model_path: Path):
try:
from piper.voice import PiperVoice # type: ignore[import]
except ImportError as exc:
raise RuntimeError(
"piper-tts is not installed. Run: pip install piper-tts"
) from exc
config_path = Path(str(model_path) + ".json")
log.info(f"Loading Piper model: {model_path}")
return PiperVoice.load(
str(model_path),
config_path=str(config_path) if config_path.exists() else None,
use_cuda=False, # ONNX runtime picks CUDA automatically when available
)
def _install(self, voice, signature: tuple | None) -> None:
"""Make *voice* the one that speaks (call with ``_lock`` held)."""
self._voice = voice
self._sample_rate = (
voice.config.sample_rate # type: ignore[union-attr]
if hasattr(voice, "config")
else 22050
)
self._loaded, self._failed = signature, None
def _check_for_new_model(self) -> None:
"""Start loading the model in the background if it changed since it was loaded."""
if self._voice is None or time.monotonic() - self._checked < self.CHECK_SECONDS:
return
self._checked = time.monotonic()
model_path = self._model_path()
if model_path is None:
return
signature = _files_signature(model_path)
if signature is None or signature in (self._loaded, self._failed):
return
if not self._reloading.acquire(blocking=False):
return # being loaded already
try:
threading.Thread(target=self._reload, args=(model_path, signature),
name="piper-reload", daemon=True).start()
except BaseException:
self._reloading.release()
raise
def _reload(self, model_path: Path, signature: tuple) -> None:
try:
voice = self._load_voice(model_path)
for _ in voice.synthesize("Bereit."): # warm up before it speaks
pass
except Exception as exc: # noqa: BLE001 - keep speaking with the loaded model
self._failed = signature
log.warning(f"Could not load the changed Piper model {model_path} ({exc}); "
"keeping the current one.")
else:
with self._lock:
self._install(voice, signature)
log.info(f"Piper model reloaded: {model_path}")
finally:
self._reloading.release()
def _model_path(self, *, warn: bool = False) -> Path | None:
"""The model to use (see the module docstring), or ``None`` if there is none."""
explicit = self.config.tts.piper_model.strip() explicit = self.config.tts.piper_model.strip()
if explicit: if explicit:
p = Path(explicit) p = Path(explicit)
if p.exists(): if p.exists():
return p return p
log.warning(f"piper_model path not found: {p}") if warn:
log.warning(f"piper_model path not found: {p}")
# Fall back to active voice profile directory. # Fall back to active voice profile directory.
candidate = (self.config.paths.voices_dir / self.config.tts.default_voice
/ "piper_model.onnx")
return candidate if candidate.exists() else None
def _resolve_model_path(self) -> Path:
"""Return the ONNX model path from config or voice profile dir."""
model_path = self._model_path(warn=True)
if model_path is not None:
return model_path
voice_name = self.config.tts.default_voice voice_name = self.config.tts.default_voice
voices_dir = self.config.paths.voices_dir voices_dir = self.config.paths.voices_dir
candidate = voices_dir / voice_name / "piper_model.onnx" candidate = voices_dir / voice_name / "piper_model.onnx"
if candidate.exists():
return candidate
from ..voices.piper_train import has_checkpoints from ..voices.piper_train import has_checkpoints
if has_checkpoints(voices_dir / voice_name): if has_checkpoints(voices_dir / voice_name):
hint = (f"Training checkpoints exist — export them with: python -m " hint = (f"Training checkpoints exist — export them with: python -m "
@@ -163,3 +245,16 @@ class PiperBackend(TTSBackend):
f"Piper ONNX model not found. Expected at {candidate} " f"Piper ONNX model not found. Expected at {candidate} "
f"or set tts.piper_model in config.yaml. {hint}" f"or set tts.piper_model in config.yaml. {hint}"
) )
def _files_signature(model_path: Path) -> tuple | None:
"""What tells a changed model apart: path, modification time and size of its files."""
try:
st = model_path.stat()
except OSError:
return None
try:
config_mtime = Path(str(model_path) + ".json").stat().st_mtime_ns
except OSError:
config_mtime = None
return str(model_path), st.st_mtime_ns, st.st_size, config_mtime
+60 -59
View File
@@ -30,6 +30,7 @@ import json
import os import os
import shutil import shutil
import sys import sys
from collections.abc import Callable
from dataclasses import dataclass, field, asdict from dataclasses import dataclass, field, asdict
from fractions import Fraction from fractions import Fraction
from pathlib import Path from pathlib import Path
@@ -106,6 +107,11 @@ _SEGMENTS_FILE = "ingest_segments.npy"
# Source audio is transcoded once to a flat PCM WAV (mono, TARGET_SR) so that # Source audio is transcoded once to a flat PCM WAV (mono, TARGET_SR) so that
# VAD and per-clip reads never decode compressed audio again. # VAD and per-clip reads never decode compressed audio again.
_CACHE_RAW = "ingest_audio_cache.i16" # raw int16 LE, mono, TARGET_SR Hz _CACHE_RAW = "ingest_audio_cache.i16" # raw int16 LE, mono, TARGET_SR Hz
# Free space that must remain on the cache drive; transcoding refuses to start
# or aborts mid-way below this.
_MIN_FREE_BYTES = 1024 * 1024 * 1024
# Re-check free space every N 10-second blocks (~10 min of audio, ~26 MB cache).
_DISK_CHECK_EVERY_BLOCKS = 60
# Save checkpoint every N processed segments (not iterations — skipped ones # Save checkpoint every N processed segments (not iterations — skipped ones
# don't count) to limit how much work is lost on an unexpected stop. # don't count) to limit how much work is lost on an unexpected stop.
_CHECKPOINT_SAVE_INTERVAL = 20 _CHECKPOINT_SAVE_INTERVAL = 20
@@ -328,55 +334,6 @@ def _is_clean_ref_clip(clip: np.ndarray, sr: int) -> bool:
return True return True
def _load_audio(path: Path, target_sr: int = TARGET_SR) -> np.ndarray:
# soundfile 0.12+ ships libsndfile 1.2.2 which reads MP3 natively.
# OGG Vorbis and FLAC were always supported.
# M4A / AAC / OPUS / WMA are not supported by libsndfile — convert to
# MP3 or WAV first.
import soundfile as sf # type: ignore[import]
info = sf.info(str(path))
sr = info.samplerate
# MP3 files with malformed ID3 tags (e.g. non-syncsafe CTOC frames) cause
# libsndfile to report INT64_MAX for the frame count. Calling sf.read()
# then tries to pre-allocate that many samples and raises a ValueError.
# Fall back to chunked streaming via sf.blocks() which reads until EOF
# without pre-allocating based on the (bogus) total-frame count.
#
# Memory note: we use a small blocksize (10 s) and resample + mono-mix
# each block immediately so that only the TARGET_SR data accumulates
# (roughly 2× smaller than keeping raw source-rate stereo chunks).
if info.frames >= sys.maxsize // 2:
log.warning(f"_load_audio: unknown frame count for {path.name} — using chunked read")
frac = Fraction(target_sr, sr).limit_denominator(1000)
resampled_chunks: list[np.ndarray] = []
for block in sf.blocks(str(path), blocksize=sr * 10, dtype="float32", always_2d=True):
# Mono-mix immediately so we don't carry stereo data any longer than needed.
mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
chunk = (
scipy.signal.resample_poly(mono, frac.numerator, frac.denominator).astype(np.float32)
if sr != target_sr else mono.astype(np.float32)
)
resampled_chunks.append(chunk)
del block, mono # free source-rate data immediately
if not resampled_chunks:
raise ValueError(f"No audio data read from {path}")
audio: np.ndarray = np.concatenate(resampled_chunks)
del resampled_chunks
return audio # already mono, already at target_sr
else:
audio, _ = sf.read(str(path), dtype="float32", always_2d=True)
if audio.ndim > 1:
audio = audio.mean(axis=1)
if sr != target_sr:
frac = Fraction(target_sr, sr).limit_denominator(1000)
audio = scipy.signal.resample_poly(audio, frac.numerator, frac.denominator).astype(np.float32)
return audio
def _segment(audio: np.ndarray, sr: int) -> list[tuple[int, int]]: def _segment(audio: np.ndarray, sr: int) -> list[tuple[int, int]]:
"""Return list of (start_sample, end_sample) speech segments via silero-vad.""" """Return list of (start_sample, end_sample) speech segments via silero-vad."""
from silero_vad import load_silero_vad, get_speech_timestamps # heavy optional dep from silero_vad import load_silero_vad, get_speech_timestamps # heavy optional dep
@@ -402,11 +359,21 @@ def _save_wav(path: Path, audio: np.ndarray, sr: int) -> None:
sf.write(str(path), audio, sr, subtype="PCM_16") sf.write(str(path), audio, sr, subtype="PCM_16")
def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None: class TranscodeCancelled(Exception):
"""Raised from the transcode thread when the caller requested a stop."""
def _transcode_to_raw(
source: Path,
target: Path,
target_sr: int,
should_stop: Callable[[], bool] | None = None,
) -> None:
"""Stream *source* → raw int16 LE binary (mono, *target_sr* Hz). """Stream *source* → raw int16 LE binary (mono, *target_sr* Hz).
Peak RAM: one 10-second block (~4 MiB). No file-size limit. Peak RAM: one 10-second block (~4 MiB). Writes atomically via a .part
Writes atomically via a .part temp file. temp file, re-checks free disk space while writing and raises
:class:`TranscodeCancelled` as soon as *should_stop* returns True.
""" """
import soundfile as sf import soundfile as sf
@@ -417,11 +384,11 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
tmp = target.with_suffix(".part") tmp = target.with_suffix(".part")
# Pre-check disk space when duration is known (skip for malformed MP3s with # Pre-check disk space when duration is known (skip for malformed MP3s with
# bogus INT64_MAX frame counts). # bogus INT64_MAX frame counts — the in-loop check below still applies).
if info.frames < sys.maxsize // 2: if info.frames < sys.maxsize // 2:
estimated_bytes = int(info.frames / sr * target_sr) * 2 # int16 = 2 B estimated_bytes = int(info.frames / sr * target_sr) * 2 # int16 = 2 B
free = shutil.disk_usage(str(target.parent)).free free = shutil.disk_usage(str(target.parent)).free
if free < estimated_bytes + 64 * 1024 * 1024: if free < estimated_bytes + _MIN_FREE_BYTES:
raise RuntimeError( raise RuntimeError(
f"Not enough disk space to cache ‘{source.name}’: " f"Not enough disk space to cache ‘{source.name}’: "
f"need ~{estimated_bytes / 1_000_000_000:.1f} GB, " f"need ~{estimated_bytes / 1_000_000_000:.1f} GB, "
@@ -429,9 +396,18 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
) )
try: try:
with open(str(tmp), "wb") as out_f: # Read until SoundFile.read() returns 0 frames instead of using
for block in sf.blocks(str(source), blocksize=sr * 10, # sf.blocks(): MP3s with malformed ID3 tags report INT64_MAX frames,
dtype="float32", always_2d=True): # and sf.blocks() ignores the real read count, so after EOF it keeps
# re-yielding its stale buffer forever (filled 800 GB once).
with sf.SoundFile(str(source)) as src, open(str(tmp), "wb") as out_f:
n_blocks = 0
while True:
if should_stop is not None and should_stop():
raise TranscodeCancelled(source.name)
block = src.read(sr * 10, dtype="float32", always_2d=True)
if len(block) == 0:
break
mono = block.mean(axis=1) if block.ndim > 1 else block.ravel() mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
if sr != target_sr: if sr != target_sr:
chunk = scipy.signal.resample_poly( chunk = scipy.signal.resample_poly(
@@ -442,6 +418,15 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
i16 = np.clip(chunk * 32767.0, -32768, 32767).astype(np.int16) i16 = np.clip(chunk * 32767.0, -32768, 32767).astype(np.int16)
out_f.write(i16.tobytes()) out_f.write(i16.tobytes())
del block, mono, chunk, i16 del block, mono, chunk, i16
n_blocks += 1
if n_blocks % _DISK_CHECK_EVERY_BLOCKS == 0:
free = shutil.disk_usage(str(target.parent)).free
if free < _MIN_FREE_BYTES:
raise RuntimeError(
f"Aborted caching ‘{source.name}’: free space on "
f"{target.parent} dropped to {free / 1_000_000_000:.1f} GB "
f"after writing {out_f.tell() / 1_000_000_000:.1f} GB"
)
tmp.replace(target) tmp.replace(target)
except Exception: except Exception:
try: try:
@@ -599,6 +584,14 @@ async def ingest_audiobook(
# peak RAM per block, no file-size limit). VAD and clip reads operate on # peak RAM per block, no file-size limit). VAD and clip reads operate on
# this cache via numpy memmap — O(1) seek, ~90 MiB peak during segmentation. # this cache via numpy memmap — O(1) seek, ~90 MiB peak during segmentation.
cache_raw = prof.dir / _CACHE_RAW cache_raw = prof.dir / _CACHE_RAW
should_stop = stop_event.is_set if stop_event is not None else None
async def _stopped_while_transcoding() -> IngestStats:
log.info(f"ingest stopped by user while transcoding {source.name}")
await _emit({"phase": "stopped", "during": "transcoding",
"at_index": 0, "total": 0})
return stats
if cp is None: if cp is None:
# Fresh run — remove any stale cache from a previous source. # Fresh run — remove any stale cache from a previous source.
if cache_raw.exists(): if cache_raw.exists():
@@ -608,7 +601,11 @@ async def ingest_audiobook(
pass pass
log.info(f"transcoding {source.name} → raw cache ...") log.info(f"transcoding {source.name} → raw cache ...")
await _emit({"phase": "loading", "source": source.name}) await _emit({"phase": "loading", "source": source.name})
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR) try:
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR,
should_stop)
except TranscodeCancelled:
return await _stopped_while_transcoding()
_cache_dur = cache_raw.stat().st_size / 2 / TARGET_SR # int16 = 2 bytes _cache_dur = cache_raw.stat().st_size / 2 / TARGET_SR # int16 = 2 bytes
log.info(f"transcoded: {_cache_dur:.1f}s") log.info(f"transcoded: {_cache_dur:.1f}s")
await _emit({"phase": "loaded", "audio_s": _cache_dur}) await _emit({"phase": "loaded", "audio_s": _cache_dur})
@@ -616,7 +613,11 @@ async def ingest_audiobook(
# Resuming but cache was deleted — retranscode. # Resuming but cache was deleted — retranscode.
log.info(f"raw cache missing — retranscoding {source.name} ...") log.info(f"raw cache missing — retranscoding {source.name} ...")
await _emit({"phase": "loading", "source": source.name}) await _emit({"phase": "loading", "source": source.name})
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR) try:
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR,
should_stop)
except TranscodeCancelled:
return await _stopped_while_transcoding()
log.info("retranscoded OK") log.info("retranscoded OK")
if cp is not None: if cp is not None:
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+495
View File
@@ -0,0 +1,495 @@
"""Coordination between Javis and a running piper training process.
Shared by the launcher (:mod:`javis.voices.piper_train`, the overnight
scheduler, the GUI) and the training process (:mod:`javis.voices.piper_compat`
and its data-loader workers), so it imports nothing but the standard library.
* **Clean stops.** The launcher names a stop file in ``JAVIS_PIPER_STOP_FILE``;
once the file exists the trainer finishes its current step, saves its
progress — the epoch so far included — and exits with :data:`EXIT_STOPPED`.
It does the same if the launcher (``JAVIS_PIPER_LAUNCHER``) has gone away,
instead of training on unsupervised.
* **Closing the console window** (or logging off, shutting down) gives the
trainer the few seconds Windows allows to save its progress:
:class:`ConsoleClose`. :func:`python_command` starts it so that none of them
is lost: without the virtual environment's ``python.exe`` launcher, which
would end it at once, and without torch's Intel Fortran console handler.
* **One run per voice.** :class:`TrainingLock` records the processes of the run
holding it, so a crashed or killed run never blocks the next one.
* **Long runs on Windows.** :func:`keep_system_awake` and
:func:`console_selection_cannot_pause`.
"""
from __future__ import annotations
import contextlib
import functools
import json
import os
import platform
import sys
import threading
import time
from pathlib import Path
STOP_FILE_ENV = "JAVIS_PIPER_STOP_FILE"
LAUNCHER_ENV = "JAVIS_PIPER_LAUNCHER" # "<pid>:<start time>" of the supervising process
# Exit codes of ``python -m javis.voices.piper_compat fit`` and of run_pipeline().
EXIT_DIVERGED = 3 # training kept producing NaN/inf: restarting would not help
EXIT_STOPPED = 4 # stopped on request, after saving a checkpoint
EXIT_BUSY = 5 # run_pipeline(): another run of this voice holds the lock
LOCK_FILE = ".training.lock"
_LEGACY_LOCK_SECONDS = 48 * 60 * 60 # locks without processes (older Javis versions)
_UNREADABLE_LOCK_SECONDS = 60 # a lock file that is being written
PARTIAL_SUFFIX = ".partial" # a file being written; renamed into place once complete
# Windows: the target is open in another process for a moment (a snapshot export
# reading last.ckpt, a virus scanner): 32/33 sharing/lock violation, 1224 mapped.
_BUSY_WINERRORS = (32, 33, 1224)
def replace_file(src, dst, *, seconds: float = 60.0) -> None:
"""``os.replace`` that gives other processes up to *seconds* to close *dst*."""
delay, deadline = 0.1, time.monotonic() + seconds
while True:
try:
os.replace(src, dst)
return
except OSError as exc:
if not (isinstance(exc, PermissionError)
or getattr(exc, "winerror", None) in _BUSY_WINERRORS) \
or time.monotonic() >= deadline:
raise
time.sleep(delay)
delay = min(delay * 2, 2.0)
# ── Processes ─────────────────────────────────────────────────────────────────
@functools.cache
def _kernel32():
import ctypes
from ctypes import wintypes
# A private instance: argtypes set on ctypes.windll.kernel32 would leak to other code.
k = ctypes.WinDLL("kernel32", use_last_error=True)
k.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD)
k.OpenProcess.restype = wintypes.HANDLE
k.GetExitCodeProcess.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD))
k.GetExitCodeProcess.restype = wintypes.BOOL
k.GetProcessTimes.argtypes = (wintypes.HANDLE,) + (ctypes.POINTER(wintypes.FILETIME),) * 4
k.GetProcessTimes.restype = wintypes.BOOL
k.CloseHandle.argtypes = (wintypes.HANDLE,)
k.CloseHandle.restype = wintypes.BOOL
k.SetThreadExecutionState.argtypes = (wintypes.DWORD,)
k.SetThreadExecutionState.restype = wintypes.DWORD
k.GetStdHandle.argtypes = (wintypes.DWORD,)
k.GetStdHandle.restype = wintypes.HANDLE
k.GetConsoleMode.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD))
k.GetConsoleMode.restype = wintypes.BOOL
k.SetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.DWORD)
k.SetConsoleMode.restype = wintypes.BOOL
k.SetConsoleCtrlHandler.argtypes = (ctypes.c_void_p, wintypes.BOOL)
k.SetConsoleCtrlHandler.restype = wintypes.BOOL
k.SetProcessShutdownParameters.argtypes = (wintypes.DWORD, wintypes.DWORD)
k.SetProcessShutdownParameters.restype = wintypes.BOOL
k.GetModuleHandleW.argtypes = (wintypes.LPCWSTR,)
k.GetModuleHandleW.restype = wintypes.HMODULE
return k
@functools.cache
def _user32():
"""``(user32, WNDPROC, WNDCLASSW)``: what a hidden window takes."""
import ctypes
from ctypes import wintypes
wndproc = ctypes.WINFUNCTYPE(wintypes.LPARAM, wintypes.HWND, wintypes.UINT,
wintypes.WPARAM, wintypes.LPARAM)
class WNDCLASSW(ctypes.Structure):
_fields_ = [("style", wintypes.UINT), ("lpfnWndProc", wndproc),
("cbClsExtra", ctypes.c_int), ("cbWndExtra", ctypes.c_int),
("hInstance", wintypes.HINSTANCE), ("hIcon", wintypes.HICON),
("hCursor", wintypes.HANDLE), ("hbrBackground", wintypes.HBRUSH),
("lpszMenuName", wintypes.LPCWSTR), ("lpszClassName", wintypes.LPCWSTR)]
u = ctypes.WinDLL("user32", use_last_error=True)
u.RegisterClassW.argtypes = (ctypes.POINTER(WNDCLASSW),)
u.RegisterClassW.restype = wintypes.ATOM
u.UnregisterClassW.argtypes = (wintypes.LPCWSTR, wintypes.HINSTANCE)
u.UnregisterClassW.restype = wintypes.BOOL
u.CreateWindowExW.argtypes = (wintypes.DWORD, wintypes.LPCWSTR, wintypes.LPCWSTR,
wintypes.DWORD, ctypes.c_int, ctypes.c_int, ctypes.c_int,
ctypes.c_int, wintypes.HWND, wintypes.HMENU,
wintypes.HINSTANCE, wintypes.LPVOID)
u.CreateWindowExW.restype = wintypes.HWND
u.DefWindowProcW.argtypes = (wintypes.HWND, wintypes.UINT, wintypes.WPARAM,
wintypes.LPARAM)
u.DefWindowProcW.restype = wintypes.LPARAM
u.GetMessageW.argtypes = (ctypes.POINTER(wintypes.MSG), wintypes.HWND, wintypes.UINT,
wintypes.UINT)
u.GetMessageW.restype = wintypes.BOOL
u.DispatchMessageW.argtypes = (ctypes.POINTER(wintypes.MSG),)
u.DispatchMessageW.restype = wintypes.LPARAM
u.PostQuitMessage.argtypes = (ctypes.c_int,)
u.PostQuitMessage.restype = None
return u, wndproc, WNDCLASSW
def python_command() -> tuple[str, dict]:
"""``(executable, extra environment)`` to start another Python of this environment.
On Windows a virtual environment's ``python.exe`` is a launcher running the
real interpreter in a job that dies with it — and the launcher ends as soon
as its console window closes, so the interpreter couldn't save anything.
Like :mod:`multiprocessing`, start the real interpreter directly;
``__PYVENV_LAUNCHER__`` tells it which environment it belongs to.
Nor does it get the console handler of the Intel Fortran runtime in torch's
``libiomp5md.dll``: on a closing window that aborts every process loading
torch ("forrtl: error (200)"), and slowly — Windows tells the processes of a
console one after the other, the newest first: data-loader workers and
exports before the trainer, which had seconds less to save its progress.
"""
if sys.platform != "win32":
return sys.executable, {}
env = {"FOR_DISABLE_CONSOLE_CTRL_HANDLER": "1"} # inherited by data-loader workers
base = getattr(sys, "_base_executable", "") or ""
if base and os.path.isfile(base) and os.path.normcase(os.path.abspath(base)) \
!= os.path.normcase(os.path.abspath(sys.executable)):
return base, {**env, "__PYVENV_LAUNCHER__": sys.executable}
return sys.executable, env
def process_start_time(pid: int) -> int | None:
"""When process *pid* started, as an opaque number; ``None`` if it isn't running.
``0`` means it runs but its start time can't be read.
"""
if pid <= 0:
return None
if sys.platform != "win32":
try:
os.kill(pid, 0) # signal 0 only checks; on Windows os.kill would terminate
except PermissionError:
return 0
except OSError:
return None
return 0
import ctypes
from ctypes import wintypes
k = _kernel32()
handle = k.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
if not handle:
return 0 if ctypes.get_last_error() == 5 else None # 5: access denied, yet it exists
try:
code = wintypes.DWORD()
if k.GetExitCodeProcess(handle, ctypes.byref(code)) and code.value != 259: # STILL_ACTIVE
return None
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
if not k.GetProcessTimes(handle, ctypes.byref(created), ctypes.byref(exited),
ctypes.byref(kernel), ctypes.byref(user)):
return 0
return (created.dwHighDateTime << 32) | created.dwLowDateTime
finally:
k.CloseHandle(handle)
def process_alive(pid: int, start_time: int | None = None) -> bool:
"""Whether *pid* is running and, given its *start_time*, is still the same process.
Windows soon hands the ID of an exited process to a new one.
"""
current = process_start_time(pid)
if current is None:
return False
return not start_time or not current or current == start_time
def this_process() -> dict:
"""``{"pid", "start"}`` of the calling process."""
pid = os.getpid()
return {"pid": pid, "start": process_start_time(pid) or 0}
def launcher_from_environment() -> tuple[int, int] | None:
"""``(pid, start time)`` of the launcher named in ``JAVIS_PIPER_LAUNCHER``, if any."""
value = os.environ.get(LAUNCHER_ENV, "")
try:
pid, start = (int(part) for part in value.split(":"))
except ValueError:
return None
return pid, start
# ── One training run per voice ────────────────────────────────────────────────
def lock_path(voice_dir: Path) -> Path:
return Path(voice_dir) / "training" / LOCK_FILE
def _read_lock(path: Path) -> str | None:
"""The lock file's text; ``None`` if there is none. Raises other ``OSError``s."""
try:
return path.read_text(encoding="utf-8")
except FileNotFoundError:
return None
def _held(path: Path, text: str) -> bool:
try:
age = time.time() - path.stat().st_mtime
except FileNotFoundError:
return False
except OSError:
return True # can't tell: assume it is held
try:
info = json.loads(text)
except ValueError:
info = None
if not isinstance(info, dict):
return age < _UNREADABLE_LOCK_SECONDS
try:
processes = [(int(p.get("pid") or 0), int(p.get("start") or 0))
for p in info.get("processes") or () if isinstance(p, dict)]
except (TypeError, ValueError):
processes = []
if not processes or info.get("host", platform.node()) != platform.node():
try:
started = float(info.get("started_epoch") or 0) or path.stat().st_mtime
except (OSError, TypeError, ValueError):
return True
return time.time() - started < _LEGACY_LOCK_SECONDS
return any(process_alive(pid, start) for pid, start in processes)
def _lock_held(path: Path) -> bool:
try:
text = _read_lock(path)
except OSError:
return True
return text is not None and _held(path, text)
def training_in_progress(voice_dir: Path) -> bool:
"""Whether a training run of this voice is running (in any process)."""
return _lock_held(lock_path(voice_dir))
class TrainingLock:
"""``training/.training.lock``: one training run per voice at a time, across processes.
Lists the launcher and, while one runs, its training process (which keeps
training for a minute or two after its launcher was killed, to save its
progress). A lock whose processes have all exited is stale and replaced.
"""
def __init__(self, voice_dir: Path) -> None:
self.path = lock_path(voice_dir)
self._info: dict | None = None
def acquire(self) -> bool:
"""Take the lock; ``False`` if another run holds it."""
info = {"host": platform.node(), "started_epoch": time.time(),
"processes": [this_process()]}
self.path.parent.mkdir(parents=True, exist_ok=True)
for _ in range(2):
try:
with open(self.path, "x", encoding="utf-8") as f:
json.dump(info, f)
except FileExistsError:
try:
text = _read_lock(self.path)
if text is not None:
if _held(self.path, text):
return False
# Left behind by a run that crashed or was killed — unless
# another launcher replaced it just now.
if _read_lock(self.path) == text:
self.path.unlink()
except FileNotFoundError:
pass
except OSError:
return False
continue
self._info = info
return True
return False
def set_trainer(self, pid: int | None) -> None:
"""Record the training process (or that none runs right now)."""
if self._info is None:
return
trainer = [{"pid": pid, "start": process_start_time(pid) or 0}] if pid else []
self._info["processes"] = self._info["processes"][:1] + trainer
with contextlib.suppress(OSError):
self.path.write_text(json.dumps(self._info), encoding="utf-8")
def release(self) -> None:
if self._info is None:
return
owner, self._info = self._info["processes"][0], None
try:
current = json.loads(self.path.read_text(encoding="utf-8"))
if (current.get("processes") or [{}])[0] == owner:
self.path.unlink()
except (OSError, ValueError, AttributeError):
pass
# ── Windows ───────────────────────────────────────────────────────────────────
@contextlib.contextmanager
def keep_system_awake():
"""Keep Windows from sleeping when idle (the display may still turn off)."""
if sys.platform != "win32":
yield
return
k = _kernel32()
es_continuous, es_system_required = 0x80000000, 0x00000001
previous = k.SetThreadExecutionState(es_continuous | es_system_required)
try:
yield
finally:
k.SetThreadExecutionState(previous if previous & es_continuous else es_continuous)
@contextlib.contextmanager
def console_selection_cannot_pause():
"""Turn off QuickEdit in a classic console window while training runs.
There, clicking into the window starts a selection that suspends every
process writing to it — training would stall until a key is pressed.
(Windows Terminal doesn't do that.)
"""
if sys.platform != "win32":
yield
return
import ctypes
from ctypes import wintypes
k = _kernel32()
handle = k.GetStdHandle(wintypes.DWORD(-10).value) # STD_INPUT_HANDLE
mode = wintypes.DWORD()
quick_edit, extended_flags = 0x0040, 0x0080
if not handle or not k.GetConsoleMode(handle, ctypes.byref(mode)) \
or not mode.value & quick_edit:
yield
return
k.SetConsoleMode(handle, (mode.value | extended_flags) & ~quick_edit)
try:
yield
finally:
k.SetConsoleMode(handle, mode.value | extended_flags)
_CONSOLE_CLOSE_EVENTS = {2: "the console window was closed", 5: "the user is logging off",
6: "the system is shutting down"}
_WM_DESTROY, _WM_QUERYENDSESSION, _WM_ENDSESSION = 0x0002, 0x0011, 0x0016
_ENDSESSION_LOGOFF = 0x80000000
# Ends before the other processes of the session — its data-loader workers
# among them: the top of the range Windows leaves to applications.
_SHUTDOWN_LEVEL = 0x3FF
class ConsoleClose:
"""Lets the training process save its progress before Windows ends it.
When its console window is closed (or the user logs off, the system shuts
down), Windows ends every process attached to it after a few seconds.
:meth:`install` handles those events: it sets :attr:`closing` — the training
loop checks it after every step — and holds the process until :meth:`done`
reports the progress saved, or *timeout* seconds have passed.
Logging off and shutting down reach a console handler only in processes
that never loaded ``user32.dll`` — Python and torch load it. A hidden
window learns of them instead (``WM_ENDSESSION``).
"""
def __init__(self, timeout: float = 30.0) -> None:
self.closing = threading.Event()
self.reason = ""
self.timeout = timeout
self._saved = threading.Event()
self._handler = None
self._window = None # the hidden window's handle
def handle(self, event: int) -> bool:
"""The console control handler; runs in a thread of its own."""
reason = _CONSOLE_CLOSE_EVENTS.get(event)
if reason is None:
return False # Ctrl+C, Ctrl+Break: left to Python's own handler
self._close(reason)
return True
def _close(self, reason: str) -> None:
self.reason = reason
self.closing.set()
self._saved.wait(self.timeout)
def done(self) -> None:
"""The progress is saved: Windows may end the process now."""
self._saved.set()
def install(self) -> None:
if sys.platform != "win32" or self._handler is not None:
return
import ctypes
from ctypes import wintypes
handler = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD)(self.handle)
if _kernel32().SetConsoleCtrlHandler(handler, True):
self._handler = handler # must outlive every call Windows may make
_kernel32().SetProcessShutdownParameters(_SHUTDOWN_LEVEL, 0)
self._watch_session_end()
def _watch_session_end(self) -> None:
"""Open the hidden window that learns of the user logging off, the system shutting down."""
ready = threading.Event()
threading.Thread(target=self._session_window, args=(ready,), name="session end",
daemon=True).start()
ready.wait(10)
def _session_window(self, ready: threading.Event) -> None:
import ctypes
from ctypes import wintypes
try:
user32, wndproc, wndclass = _user32()
instance = _kernel32().GetModuleHandleW(None)
def window_procedure(hwnd, message, wparam, lparam):
if message == _WM_QUERYENDSESSION:
return 1 # may end: the progress gets saved at WM_ENDSESSION
if message == _WM_ENDSESSION:
if wparam: # else cancelled
self._close("the user is logging off" if lparam & _ENDSESSION_LOGOFF
else "the system is shutting down")
return 0
if message == _WM_DESTROY:
user32.PostQuitMessage(0)
return 0
return user32.DefWindowProcW(hwnd, message, wparam, lparam)
procedure = wndproc(window_procedure) # referenced while the window exists
name = f"JavisTrainingSession{os.getpid()}-{id(self):x}"
if not user32.RegisterClassW(ctypes.byref(wndclass(
lpfnWndProc=procedure, hInstance=instance, lpszClassName=name))):
return
# Top-level but never shown: message-only windows get no WM_ENDSESSION.
window = user32.CreateWindowExW(0, name, "Javis voice training", 0, 0, 0, 0, 0,
None, None, instance, None)
if not window:
user32.UnregisterClassW(name, instance)
return
self._window = window
finally:
ready.set()
message = wintypes.MSG()
while user32.GetMessageW(ctypes.byref(message), None, 0, 0) > 0:
user32.DispatchMessageW(ctypes.byref(message))
user32.UnregisterClassW(name, instance)
+26 -79
View File
@@ -18,31 +18,29 @@ Design notes
without a restart. without a restart.
* **Window guard.** Training will not *start* if fewer than * **Window guard.** Training will not *start* if fewer than
``min_remaining_minutes`` of the window are left. A run that has already ``min_remaining_minutes`` of the window are left. A run that has already
started is allowed to run to completion — PyTorch-Lightning cannot be started keeps going; quitting Javis stops it cleanly — the training
cleanly killed mid-epoch, and an unfinished run is safe: it leaves a process saves its progress after its current step (the epoch so far
checkpoint that the next run resumes from. included) first, so Javis's process can take a little while to exit —
* **Concurrency guard.** A lock file (``training/.training.lock``) prevents and the next run resumes from where it stopped. While it runs, the
the scheduler and the interactive GUI button from training the same voice voice's model is updated with its progress (see ``piper_train``).
at the same time (they run in separate processes). * **Concurrency guard.** ``run_pipeline`` holds a lock file
(``training/.training.lock``) naming its processes, so the scheduler and
the interactive GUI button (separate processes) never train the same voice
at the same time; a run that crashed or was killed leaves no lasting lock.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import json
import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from ..core.config import Config from ..core.config import Config
from ..core.logger import get_logger from ..core.logger import get_logger
from .training_control import EXIT_BUSY, EXIT_STOPPED, training_in_progress
log = get_logger("voices.training") log = get_logger("voices.training")
# A lock is considered "live" for this long. Overnight runs take a few
# hours; 48 h comfortably bounds a stuck/crashed run so it self-expires.
_LOCK_STALE_SECONDS = 48 * 60 * 60
# ── Pure helpers (unit-testable) ───────────────────────────────────────────── # ── Pure helpers (unit-testable) ─────────────────────────────────────────────
@@ -97,62 +95,10 @@ def window_end(now: datetime, start_hour: int, end_hour: int) -> datetime:
return end return end
# ── Cross-process lock ────────────────────────────────────────────────────────
def _lock_path(voice_dir: Path) -> Path:
return voice_dir / "training" / ".training.lock"
def _lock_alive(voice_dir: Path) -> bool:
"""True if a recent lock file indicates another training is in progress."""
lp = _lock_path(voice_dir)
if not lp.exists():
return False
try:
data = json.loads(lp.read_text(encoding="utf-8"))
ts = float(data.get("started_epoch", 0.0))
except Exception:
try:
ts = lp.stat().st_mtime
except OSError:
return False
if not ts:
return False
return (time.time() - ts) < _LOCK_STALE_SECONDS
def _acquire_lock(voice_dir: Path) -> bool:
"""Atomically claim the training lock. Returns False if already held."""
lp = _lock_path(voice_dir)
lp.parent.mkdir(parents=True, exist_ok=True)
try:
# 'x' = exclusive create; fails if a live lock already exists.
with open(lp, "x", encoding="utf-8") as f:
f.write(json.dumps({"started_epoch": time.time()}))
return True
except FileExistsError:
return False
except OSError:
# On platforms where 'x' is flaky, fall back to the liveness check.
return not _lock_alive(voice_dir)
def _release_lock(voice_dir: Path) -> None:
try:
_lock_path(voice_dir).unlink(missing_ok=True)
except OSError:
pass
def training_in_progress(voice_dir: Path) -> bool:
"""Public helper: is another training of this voice currently running?"""
return _lock_alive(voice_dir)
# ── One run ─────────────────────────────────────────────────────────────────── # ── One run ───────────────────────────────────────────────────────────────────
async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = None) -> None: async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = None) -> None:
"""Execute a single scheduled training run (threaded) with locking.""" """Execute a single scheduled training run (threaded)."""
from .piper_train import run_pipeline # imported lazily (heavy deps) from .piper_train import run_pipeline # imported lazily (heavy deps)
t = config.training t = config.training
@@ -166,14 +112,10 @@ async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = Non
f"{t.min_remaining_minutes} min minimum); skipping start.") f"{t.min_remaining_minutes} min minimum); skipping start.")
return return
if _lock_alive(voice_dir): if training_in_progress(voice_dir):
log.info("Another training of this voice is already running; skipping.") log.info("Another training of this voice is already running; skipping.")
return return
if not _acquire_lock(voice_dir):
log.info("Could not acquire training lock (already held); skipping.")
return
voice = voice_dir.name voice = voice_dir.name
n_clips = 0 n_clips = 0
meta = voice_dir / "training" / "metadata.csv" meta = voice_dir / "training" / "metadata.csv"
@@ -214,25 +156,30 @@ async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = Non
try: try:
rc = await asyncio.shield(work) rc = await asyncio.shield(work)
except asyncio.CancelledError: except asyncio.CancelledError:
# Javis is quitting: stop the child process so the worker thread (and # Javis is quitting: have the training process save a checkpoint and
# interpreter exit) doesn't block until training finishes. # stop, so the worker thread (and interpreter exit) isn't kept waiting
from .piper_train import request_stop # until training finishes.
from .piper_train import STOP_GRACE_SECONDS, request_stop
request_stop() request_stop()
await asyncio.wait({work}, timeout=60) await asyncio.wait({work}, timeout=STOP_GRACE_SECONDS + 30)
raise raise
except Exception as exc: # noqa: BLE001 — training must never kill the loop except Exception as exc: # noqa: BLE001 — training must never kill the loop
log.exception(f"Scheduled training crashed: {exc}") log.exception(f"Scheduled training crashed: {exc}")
rc = -1 rc = -1
finally:
if work.done():
_release_lock(voice_dir)
status = "completed" if rc == 0 else f"FAILED (exit {rc})" if rc == 0:
status = "completed"
elif rc == EXIT_STOPPED:
status = "stopped (progress saved)"
elif rc == EXIT_BUSY:
status = "skipped (this voice is being trained already)"
else:
status = f"FAILED (exit {rc})"
log.info(f"Scheduled training {status}: voice={voice!r}") log.info(f"Scheduled training {status}: voice={voice!r}")
if bus is not None: if bus is not None:
await _safe_publish( await _safe_publish(
bus, "task_done", bus, "task_done",
{"id": "training", "ok": rc == 0, {"id": "training", "ok": rc in (0, EXIT_STOPPED, EXIT_BUSY),
"msg": f"Voice training {status}"}) "msg": f"Voice training {status}"})
+123
View File
@@ -0,0 +1,123 @@
"""Audiobook ingest: raw-cache transcoding must terminate and clean up."""
from __future__ import annotations
import asyncio
import sys
import tempfile
import threading
import unittest
from collections import namedtuple
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, PropertyMock, patch
import numpy as np
SRC_SR = 44100
SRC_SECONDS = 25 # two full 10 s blocks + one partial block
TARGET_SR = 22050
_DiskUsage = namedtuple("_DiskUsage", "total used free")
class TranscodeToRawTests(unittest.TestCase):
def setUp(self) -> None:
try:
import soundfile as sf
except ImportError:
self.skipTest("soundfile not installed")
from javis.voices import audiobook_ingest
self.sf = sf
self.ingest = audiobook_ingest
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.source = self.root / "book.wav"
t = np.arange(SRC_SR * SRC_SECONDS) / SRC_SR
tone = 0.3 * np.sin(2 * np.pi * 220 * t)
sf.write(str(self.source), np.stack([tone, tone], axis=1), SRC_SR, subtype="PCM_16")
self.target = self.root / "cache" / "ingest_audio_cache.i16"
def tearDown(self) -> None:
self._tmp.cleanup()
def _bogus_frame_count(self):
"""Mimic an MP3 with a malformed ID3 tag: INT64_MAX frames, unseekable."""
frames = patch.object(self.sf.SoundFile, "frames", new_callable=PropertyMock,
return_value=sys.maxsize)
seekable = patch.object(self.sf.SoundFile, "seekable", return_value=False)
return frames, seekable
def _run_with_timeout(self, timeout: float = 60.0, **kw) -> BaseException | None:
errors: list[BaseException] = []
def run():
try:
self.ingest._transcode_to_raw(self.source, self.target, TARGET_SR, **kw)
except BaseException as e: # noqa: BLE001
errors.append(e)
th = threading.Thread(target=run, daemon=True)
th.start()
th.join(timeout)
self.assertFalse(th.is_alive(), "transcode did not stop at EOF (runaway)")
return errors[0] if errors else None
def test_unknown_frame_count_stops_at_eof(self) -> None:
frames, seekable = self._bogus_frame_count()
with frames, seekable:
self.assertGreaterEqual(self.sf.info(str(self.source)).frames, sys.maxsize // 2)
err = self._run_with_timeout()
self.assertIsNone(err)
self.assertEqual(self.target.stat().st_size, SRC_SECONDS * TARGET_SR * 2)
self.assertFalse(self.target.with_suffix(".part").exists())
def test_stop_request_aborts_and_removes_partial_file(self) -> None:
err = self._run_with_timeout(should_stop=lambda: True)
self.assertIsInstance(err, self.ingest.TranscodeCancelled)
self.assertFalse(self.target.exists())
self.assertFalse(self.target.with_suffix(".part").exists())
def test_disk_running_full_aborts_and_removes_partial_file(self) -> None:
frames, seekable = self._bogus_frame_count()
full = patch.object(self.ingest.shutil, "disk_usage",
return_value=_DiskUsage(10**12, 10**12, 0))
every_block = patch.object(self.ingest, "_DISK_CHECK_EVERY_BLOCKS", 1)
with frames, seekable, full, every_block:
err = self._run_with_timeout()
self.assertIsInstance(err, RuntimeError)
self.assertIn("free space", str(err))
self.assertFalse(self.target.exists())
self.assertFalse(self.target.with_suffix(".part").exists())
def test_insufficient_space_for_known_length_fails_fast(self) -> None:
with patch.object(self.ingest.shutil, "disk_usage",
return_value=_DiskUsage(10**12, 10**12, 0)):
err = self._run_with_timeout()
self.assertIsInstance(err, RuntimeError)
self.assertIn("Not enough disk space", str(err))
self.assertFalse(self.target.with_suffix(".part").exists())
def test_stop_button_during_transcode_ends_ingest_as_stopped(self) -> None:
events: list[dict] = []
class Bus:
async def publish(self, topic, payload):
events.append(payload)
mgr = MagicMock()
mgr.root = self.root
mgr.create.return_value = SimpleNamespace(dir=self.root / "voice")
stop = asyncio.Event()
stop.set()
with patch("javis.voices.manager.VoiceManager", return_value=mgr):
stats = asyncio.run(self.ingest.ingest_audiobook(
None, self.source, "voice", bus=Bus(), stop_event=stop, resume=False))
self.assertEqual(stats.total_segments, 0)
self.assertEqual(events[-1]["phase"], "stopped")
self.assertEqual(events[-1]["during"], "transcoding")
self.assertFalse((self.root / "voice" / "ingest_audio_cache.i16").exists())
self.assertFalse((self.root / "voice" / "ingest_audio_cache.part").exists())
if __name__ == "__main__":
unittest.main()
+120
View File
@@ -0,0 +1,120 @@
"""Slave GUI chat: newest-first transcript and fullscreen chat mode (offscreen Qt)."""
from __future__ import annotations
import os
import sys
from pathlib import Path
import pytest
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
SLAVE_SRC = Path(__file__).resolve().parents[1] / "javis-slave" / "src"
if str(SLAVE_SRC) not in sys.path:
sys.path.insert(0, str(SLAVE_SRC))
QtWidgets = pytest.importorskip("PySide6.QtWidgets")
QtTest = pytest.importorskip("PySide6.QtTest")
from javis_slave import main_window # noqa: E402
@pytest.fixture(scope="module")
def qapp():
return QtWidgets.QApplication.instance() or QtWidgets.QApplication([])
@pytest.fixture
def screen(qapp):
scr = main_window.MainScreen()
scr.resize(900, 700)
scr.show()
yield scr
scr.close()
def _texts(scr) -> list[str]:
"""Bubble texts from the top of the transcript to the bottom."""
layout = scr.transcript_layout
out = []
for i in range(layout.count()):
row = layout.itemAt(i).widget()
if row is not None:
out.append(row.findChild(QtWidgets.QLabel).text())
return out
def test_newest_message_is_on_top(screen):
screen.set_turn("frage 1", "antwort 1")
screen.set_turn("frage 2", "antwort 2")
screen.show_error("kaputt")
assert _texts(screen) == [
"⚠ kaputt", "antwort 2", "frage 2", "antwort 1", "frage 1"]
def test_prune_drops_oldest_at_the_bottom(screen):
for i in range(30):
screen.set_turn(f"u{i}", f"a{i}")
texts = _texts(screen)
assert len(texts) == 40
assert texts[0] == "a29"
assert "u0" not in texts and "u29" in texts
def test_scrolls_to_top_on_new_message(screen):
for i in range(12):
screen.set_turn(f"u{i}", "x " * 200)
QtTest.QTest.qWait(150) # lets the layout settle (and the first auto-scroll fire)
vbar = screen.transcript_scroll.verticalScrollBar()
assert vbar.maximum() > 0
vbar.setValue(vbar.maximum())
screen.set_turn("neu", "neu")
QtTest.QTest.qWait(150) # the scroll is deferred by a 50 ms timer
assert vbar.value() == vbar.minimum()
def test_fullscreen_button_goes_fullscreen_and_keeps_orb_and_chat_visible(qapp):
win = main_window.SlaveWindow()
win.show_main()
win.show()
scr = win.main_screen
scr.set_turn("frage", "antwort")
try:
assert not win.isFullScreen()
assert scr.expand_btn.toolTip() == "Vollbild"
scr.expand_btn.click()
QtTest.QTest.qWait(50)
assert win.isFullScreen()
assert scr.orb.isVisible() and scr.transcript_scroll.isVisible()
assert scr.expand_btn.toolTip() == "Vollbild beenden"
scr.expand_btn.click()
QtTest.QTest.qWait(50)
assert not win.isFullScreen()
assert scr.orb.isVisible() and scr.transcript_scroll.isVisible()
assert scr.expand_btn.toolTip() == "Vollbild"
finally:
win.close()
def test_esc_and_f11_keep_button_in_sync(qapp):
win = main_window.SlaveWindow()
win.show_main()
win.show()
try:
win._toggle_fullscreen() # F11
QtTest.QTest.qWait(50)
assert win.isFullScreen() and win.main_screen.expand_btn.toolTip() == "Vollbild beenden"
win._leave_fullscreen() # Esc
QtTest.QTest.qWait(50)
assert not win.isFullScreen() and win.main_screen.expand_btn.toolTip() == "Vollbild"
finally:
win.close()
def test_expand_icons_render(qapp):
for kind in ("expand", "collapse"):
img = main_window._icon(kind, "#cccccc", 26).pixmap(26, 26).toImage()
assert any(img.pixelColor(x, y).alpha() > 0
for x in range(img.width()) for y in range(img.height()))
+335
View File
@@ -0,0 +1,335 @@
"""Unidentified-speaker inbox, sorting clips into profiles, and live samples."""
from __future__ import annotations
import asyncio
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import numpy as np
from javis.core.config import Config, PathsConfig
from javis.core.events import EventBus
from javis.core.state import AgentState
from javis.stt.unidentified import UnidentifiedStore
from javis.stt.voice_profiler import NullProfiler, VoiceProfiler
SR = 16000
def _clip(voice: int, seconds: float = 1.0) -> np.ndarray:
"""Audio whose first sample encodes the (fake) speaker id."""
a = np.full(int(SR * seconds), 0.2, dtype=np.float32)
a[0] = voice / 100.0
return a
def _fake_embed(audio: np.ndarray, sr: int) -> np.ndarray:
voice = round(float(audio[0]) * 100)
v = np.full(192, 0.01, dtype=np.float32)
v[voice] = 1.0
return v / np.linalg.norm(v)
def _profiler(base: Path) -> VoiceProfiler:
prof = VoiceProfiler(Config(paths=PathsConfig(base_dir=base)))
prof._encoder = object() # pretend the ECAPA model is loaded
prof.embed = _fake_embed # type: ignore[method-assign]
return prof
class UnidentifiedStoreTests(unittest.TestCase):
def test_add_list_load_remove_and_persist(self) -> None:
with tempfile.TemporaryDirectory() as d:
store = UnidentifiedStore(Path(d), max_clips=10)
emb = _fake_embed(_clip(3), SR)
a = store.add(_clip(3), SR, embedding=emb, text=" hallo ", source="local",
best_name="Bob", best_score=0.41)
b = store.add(_clip(4, 2.0), SR, text="zwei")
entries = store.entries()
self.assertEqual([e["id"] for e in entries], [b, a]) # newest first
self.assertEqual(entries[1]["text"], "hallo")
self.assertEqual(entries[1]["best_name"], "Bob")
self.assertAlmostEqual(entries[0]["duration_s"], 2.0)
audio, sr = store.load_audio(a)
self.assertEqual(sr, SR)
self.assertEqual(len(audio), SR)
np.testing.assert_allclose(store.load_embedding(a), emb, rtol=1e-6)
self.assertIsNone(store.load_embedding(b))
# A fresh instance reads the same index from disk.
self.assertEqual(len(UnidentifiedStore(Path(d))), 2)
self.assertEqual(store.remove([a, "nope"]), 1)
self.assertFalse((Path(d) / f"{a}.wav").exists())
self.assertFalse((Path(d) / f"{a}.npy").exists())
self.assertEqual(store.clear(), 1)
self.assertEqual(store.entries(), [])
with self.assertRaises(KeyError):
store.load_audio(a)
def test_oldest_clips_are_pruned(self) -> None:
with tempfile.TemporaryDirectory() as d:
store = UnidentifiedStore(Path(d), max_clips=2)
ids = [store.add(_clip(i + 1), SR, text=str(i)) for i in range(3)]
self.assertEqual([e["id"] for e in store.entries()], [ids[2], ids[1]])
self.assertFalse((Path(d) / f"{ids[0]}.wav").exists())
class ProfilerEmbeddingTests(unittest.TestCase):
def test_enroll_rank_identify_refine(self) -> None:
with tempfile.TemporaryDirectory() as d:
prof = _profiler(Path(d))
self.assertTrue(prof.enroll_embeddings("Alice", [_fake_embed(_clip(1), SR)] * 2))
prof.enroll("Bob", [_clip(2)], SR)
ranked = prof.rank(_fake_embed(_clip(1), SR))
self.assertEqual(ranked[0][0], "Alice")
self.assertGreater(ranked[0][1], 0.99)
self.assertEqual(prof.identify_embedding(_fake_embed(_clip(2), SR))[0], "Bob")
self.assertIsNone(prof.identify_embedding(_fake_embed(_clip(9), SR)))
self.assertEqual(prof.rank(np.zeros(192)), [])
# Running centroid: a Bob sample pulls Alice's centroid towards Bob.
before = dict(prof.rank(_fake_embed(_clip(2), SR)))["Alice"]
self.assertTrue(prof.refine_embedding("Alice", _fake_embed(_clip(2), SR)))
after = dict(prof.rank(_fake_embed(_clip(2), SR)))["Alice"]
self.assertGreater(after, before)
alice = next(p for p in prof.list_profiles() if p["name"] == "Alice")
self.assertEqual(alice["clip_count"], 3)
self.assertFalse(prof.refine_embedding("Nobody", _fake_embed(_clip(1), SR)))
self.assertFalse(prof.refine_embedding("Alice", np.ones(10))) # wrong shape
self.assertFalse(prof.enroll_embeddings("Zero", [np.zeros(192)]))
def test_sanitised_name_collision_gets_its_own_file(self) -> None:
with tempfile.TemporaryDirectory() as d:
prof = _profiler(Path(d))
prof.enroll_embeddings("Anna B", [_fake_embed(_clip(1), SR)])
prof.enroll_embeddings("Anna_B", [_fake_embed(_clip(2), SR)])
files = {p["name"]: p["file"] for p in prof.list_profiles()}
self.assertNotEqual(files["Anna B"], files["Anna_B"])
prof2 = _profiler(Path(d))
prof2._load_index()
self.assertEqual(prof2.rank(_fake_embed(_clip(1), SR))[0][0], "Anna B")
def test_null_profiler_mirrors_api(self) -> None:
n = NullProfiler()
self.assertFalse(n.is_ready())
self.assertFalse(n.has_profile("x"))
self.assertEqual(n.rank(np.ones(192)), [])
self.assertFalse(n.enroll_embeddings("x", [np.ones(192)]))
self.assertFalse(n.refine_embedding("x", np.ones(192)))
self.assertIsNone(n.identify_embedding(np.ones(192)))
class _FakeMic:
chunk_ms = 32
def __init__(self) -> None:
self.drains = 0
def drain(self) -> None:
self.drains += 1
def _orchestrator(base: Path):
from javis.agent.orchestrator import Orchestrator
orch = object.__new__(Orchestrator)
orch.config = Config(paths=PathsConfig(base_dir=base))
orch.bus = EventBus()
orch.profiler = _profiler(base)
orch.unidentified = UnidentifiedStore(base / "voice_profiles" / "unidentified")
orch._enroll_clip_buffer = []
orch._enroll_ask_pending = False
orch._turn_lock = asyncio.Lock()
orch._mic_free = asyncio.Event()
orch._mic_free.set()
orch.mic = _FakeMic()
orch.vad = None
orch.state = AgentState.IDLE
events: list[tuple[str, object]] = []
async def _record(name, payload=None):
events.append((name, payload))
for ev in ("unidentified_changed", "voice_profiles_changed", "speaker_enrolled",
"speaker_ask_name"):
orch.bus.subscribe(ev, lambda p, _n=ev: _record(_n, p))
return orch, events
class AssignUnidentifiedTests(unittest.IsolatedAsyncioTestCase):
async def test_unknown_turn_is_stored_then_sorted_into_new_profile(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, events = _orchestrator(Path(d))
orch.config.speaker_id.auto_enroll = False
await orch._handle_unknown_speaker(_clip(5), SR, "wer bin ich", source="Küche")
await orch._handle_unknown_speaker(_clip(5), SR, " ", source="local") # no text
await asyncio.sleep(0.2)
entries = orch.unidentified.entries()
self.assertEqual(len(entries), 1)
self.assertEqual(entries[0]["source"], "Küche")
self.assertIn(("unidentified_changed", {"added": entries[0]["id"]}), events)
res = await orch.assign_unidentified([entries[0]["id"]], " Carla ")
self.assertEqual(res, {"name": "Carla", "added": 1, "created": True})
self.assertTrue(orch.profiler.has_profile("Carla"))
self.assertEqual(orch.unidentified.entries(), [])
self.assertEqual(orch.profiler.identify_embedding(_fake_embed(_clip(5), SR))[0],
"Carla")
names = [e[0] for e in events]
self.assertIn("voice_profiles_changed", names)
async def test_assign_to_existing_profile_refines_it(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, _ = _orchestrator(Path(d))
orch.profiler.enroll("Dora", [_clip(6)], SR)
ids = [orch.unidentified.add(_clip(6), SR, text="a"),
orch.unidentified.add(_clip(6), SR, text="b")] # no stored embedding
res = await orch.assign_unidentified(ids, "Dora")
self.assertEqual(res["added"], 2)
self.assertFalse(res["created"])
dora = next(p for p in orch.profiler.list_profiles() if p["name"] == "Dora")
self.assertEqual(dora["clip_count"], 3)
async def test_assign_validates_input(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, _ = _orchestrator(Path(d))
with self.assertRaises(ValueError):
await orch.assign_unidentified(["x"], " ")
with self.assertRaises(RuntimeError):
await orch.assign_unidentified(["missing"], "Eve")
orch.profiler._encoder = None
with self.assertRaises(RuntimeError):
await orch.assign_unidentified(["x"], "Eve")
async def test_name_prompt_confirm_removes_inbox_copies(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, events = _orchestrator(Path(d))
await orch._handle_unknown_speaker(_clip(7), SR, "hallo", source="local")
await orch._handle_unknown_speaker(_clip(7), SR, "noch mal", source="local")
self.assertEqual(len(orch.pending_enroll_clips()), 2)
self.assertEqual(sum(1 for e in events if e[0] == "speaker_ask_name"), 1)
self.assertTrue(await orch.auto_enroll_confirm("k", "Frank"))
self.assertEqual(orch.unidentified.entries(), [])
self.assertTrue(orch.profiler.has_profile("Frank"))
self.assertIn(("speaker_enrolled", {"name": "Frank", "clip_count": 2}), events)
async def test_dismiss_keeps_inbox_and_allows_new_prompt(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, events = _orchestrator(Path(d))
await orch._handle_unknown_speaker(_clip(8), SR, "hallo", source="local")
await asyncio.sleep(0.2)
orch.auto_enroll_dismiss()
self.assertEqual(orch.pending_enroll_clips(), [])
self.assertEqual(len(orch.unidentified.entries()), 1)
await orch._handle_unknown_speaker(_clip(8), SR, "wieder", source="local")
self.assertEqual(sum(1 for e in events if e[0] == "speaker_ask_name"), 2)
class LiveSampleTests(unittest.IsolatedAsyncioTestCase):
async def test_record_and_add_samples(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, events = _orchestrator(Path(d))
orch.profiler.enroll("Gina", [_clip(9)], SR)
seen = {}
async def fake_record(config, mic, vad, **kw):
# The wake loop must be paused and the turn lock held meanwhile.
seen["mic_free"] = orch._mic_free.is_set()
seen["locked"] = orch._turn_lock.locked()
seen["state"] = orch.state
return _clip(9)
async def no_chime():
return None
orch._play_wake_chime = no_chime
with patch("javis.agent.orchestrator.record_utterance", fake_record):
sample = await orch.record_voice_sample("Gina")
self.assertEqual(seen, {"mic_free": False, "locked": True,
"state": AgentState.ENROLLING})
self.assertTrue(orch._mic_free.is_set())
self.assertFalse(orch._turn_lock.locked())
self.assertEqual(orch.state, AgentState.IDLE)
self.assertGreater(sample.score, 0.99)
self.assertEqual(sample.best_name, "Gina")
self.assertAlmostEqual(sample.duration_s, 1.0)
used = await orch.add_voice_samples("Gina", [sample, sample])
self.assertEqual(used, 2)
gina = next(p for p in orch.profiler.list_profiles() if p["name"] == "Gina")
self.assertEqual(gina["clip_count"], 3)
self.assertIn(("voice_profiles_changed",
{"name": "Gina", "added": 2, "created": False}), events)
async def test_record_errors_are_user_facing(self) -> None:
with tempfile.TemporaryDirectory() as d:
orch, _ = _orchestrator(Path(d))
async def no_speech(*a, **kw):
return None
async def no_chime():
return None
orch._play_wake_chime = no_chime
with patch("javis.agent.orchestrator.record_utterance", no_speech), \
self.assertRaisesRegex(RuntimeError, "No speech"):
await orch.record_voice_sample("Gina")
self.assertTrue(orch._mic_free.is_set())
self.assertFalse(orch._turn_lock.locked())
async with orch._turn_lock: # a conversation turn is running
with self.assertRaisesRegex(RuntimeError, "busy"):
async with orch._exclusive_mic(wait_s=0.05):
pass
self.assertTrue(orch._mic_free.is_set())
orch.mic = None
with self.assertRaisesRegex(RuntimeError, "microphone"):
await orch.record_voice_sample("Gina")
async def test_wake_loop_yields_mic_to_exclusive_recording(self) -> None:
from javis.agent.orchestrator import Orchestrator
with tempfile.TemporaryDirectory() as d:
orch, _ = _orchestrator(Path(d))
orch._stop = asyncio.Event()
fed: list[int] = []
class Wake:
def reset(self):
pass
def feed(self, chunk):
fed.append(len(chunk))
return 0.0
def triggered(self, score):
return False
class Mic(_FakeMic):
async def chunks(self, stale_timeout=None):
while True:
await asyncio.sleep(0.005)
yield np.zeros(512, dtype=np.float32)
orch.wake = Wake()
orch.mic = Mic()
orch._mic_free.clear()
task = asyncio.ensure_future(Orchestrator._wait_for_wake(orch))
await asyncio.sleep(0.05)
self.assertEqual(fed, []) # waits while the mic is taken
orch._mic_free.set()
await asyncio.sleep(0.05)
self.assertTrue(fed) # resumed wake detection
orch._mic_free.clear()
self.assertFalse(await asyncio.wait_for(task, 1.0))
if __name__ == "__main__":
unittest.main()
File diff suppressed because it is too large. Load diff