ji
This commit is contained in:
1 parent
4c16f87a3e
commit
2493c5c532
27 files changed
+8008
-568
No files matched your search
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"entries": {
|
||||||
|
"spotify_pause_access_denied_unknown_speaker": {
|
||||||
|
"value": "spotify__pausePlayback mit deviceId 'a169794694f72b88e60439a45e2b506a30254979' faellt mit 'Access denied: unknown speaker' durch. Bei Spotify-Befehlern zuerst die Integration laden oder gueltige Geraete-IDs ueber getAvailableDevices ermitteln.",
|
||||||
|
"category": "spotify",
|
||||||
|
"created_at": "2026-10-10T10:43:30.486067+00:00",
|
||||||
|
"updated_at": "2026-10-10T10:43:30.486067+00:00"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"entries": {
|
||||||
|
"user_name": {
|
||||||
|
"value": "Der Nutzer heisst Florian.",
|
||||||
|
"category": "personal",
|
||||||
|
"created_at": "2026-10-10T10:40:14.388969+00:00",
|
||||||
|
"updated_at": "2026-10-10T10:40:14.388969+00:00"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,22 @@
|
|||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
"commands": {}
|
"commands": {
|
||||||
|
"abfbc899-8ad1-436b-b386-152c80669dcd": {
|
||||||
|
"id": "abfbc899-8ad1-436b-b386-152c80669dcd",
|
||||||
|
"trigger": "küche ein",
|
||||||
|
"match_mode": "contains",
|
||||||
|
"actions": [
|
||||||
|
{
|
||||||
|
"tool": "homey__control_zone",
|
||||||
|
"args": {
|
||||||
|
"capability": "onoff",
|
||||||
|
"value": true,
|
||||||
|
"zone": "Küche"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"created_at": "2026-10-10T10:39:38.382223+00:00",
|
||||||
|
"updated_at": "2026-10-10T10:39:38.382223+00:00"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"devices": {
|
||||||
|
"a32f0630-04e5-4eda-9ae2-44fa4eeb6068": {
|
||||||
|
"name": "Tablet K\u00fcche",
|
||||||
|
"api_key_hash": "89db0c92e376e0160a756f8285d940ba8813d2867cd699badac4deb2b85b813b",
|
||||||
|
"room": "K\u00fcche",
|
||||||
|
"ip_address": "10.10.10.225",
|
||||||
|
"created_at": "2026-10-07T17:40:24+00:00",
|
||||||
|
"last_seen": "2026-10-10T10:44:31+00:00",
|
||||||
|
"last_seen_ip": "10.10.10.225"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "Florian",
|
||||||
|
"file": "Florian.npy",
|
||||||
|
"enrolled_at": "2026-10-07T21:32:39",
|
||||||
|
"clip_count": 13,
|
||||||
|
"last_seen": "2026-10-10T12:44:06"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
@@ -89,6 +89,33 @@ Push-to-talk: `Ctrl+Space` in the window, `Ctrl+Alt+Space` system-wide (Windows)
|
|||||||
3. Javis will VAD-split, transcribe, and build a clean dataset.
|
3. Javis will VAD-split, transcribe, and build a clean dataset.
|
||||||
4. Test with zero-shot cloning (instant) or click **Fine-tune** for higher quality.
|
4. Test with zero-shot cloning (instant) or click **Fine-tune** for higher quality.
|
||||||
|
|
||||||
|
Fine-tuning runs in a console window of its own. Meanwhile Javis speaks with the
|
||||||
|
voice being trained, refreshed with the training progress every 15 minutes. Stop it
|
||||||
|
any time — Ctrl+C, closing the window, logging off or shutting down saves the
|
||||||
|
epoch so far, and the next run continues with its next batch. A crash loses at most
|
||||||
|
the last 10 minutes (`JAVIS_PIPER_PROGRESS_MINUTES`); training restarts by itself.
|
||||||
|
|
||||||
|
## Speaker identification
|
||||||
|
|
||||||
|
With `speaker_id.enabled`, Javis recognises who is talking (ECAPA embeddings) and manages
|
||||||
|
speakers in **Intelligence → Voice Profiles**:
|
||||||
|
|
||||||
|
- **Enroll** — `+` records a few sentences and creates a profile.
|
||||||
|
- **Unidentified speakers** — utterances that matched no profile are kept (audio,
|
||||||
|
transcript, time, source device, closest profile). **▶ Play** listens to one;
|
||||||
|
**Sort…** adds it to an existing profile or creates a new one. Recordings that sound
|
||||||
|
alike are offered and pre-selected so you can sort several at once. ✕ / **Clear all**
|
||||||
|
delete them.
|
||||||
|
- **Enhance a voice** — 🎤 on a profile card opens a recorder: click **● Record sample**,
|
||||||
|
speak, review each sample (playback + how well it matches the profile; mismatches are
|
||||||
|
flagged), then **Add N samples** to blend them in.
|
||||||
|
- The "unknown speaker" pop-up (`speaker_id.auto_enroll`) can also play back what was said.
|
||||||
|
|
||||||
|
Unknown-speaker clips are stored locally only, in `<voice_profiles_dir>/unidentified/`.
|
||||||
|
They are capped by `speaker_id.max_unidentified` (default 50, oldest dropped first). Turn
|
||||||
|
the inbox off with `speaker_id.keep_unidentified: false`. Both settings are under
|
||||||
|
**Settings → Speaker Identification**.
|
||||||
|
|
||||||
## MCP servers
|
## MCP servers
|
||||||
|
|
||||||
Edit `mcp_servers.json`. Schema matches VS Code's `mcp.json`, so you can copy/paste server entries.
|
Edit `mcp_servers.json`. Schema matches VS Code's `mcp.json`, so you can copy/paste server entries.
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ training:
|
|||||||
batch_size: 32
|
batch_size: 32
|
||||||
num_workers: 2
|
num_workers: 2
|
||||||
accelerator: "gpu" # gpu | cpu
|
accelerator: "gpu" # gpu | cpu
|
||||||
precision: "16-mixed" # 32 | 16-mixed | bf16-mixed
|
precision: "bf16-mixed" # bf16-mixed (fastest) | 32 | 16-mixed
|
||||||
language: "de" # espeak-ng voice code used for phonemisation (de, en-us, …)
|
language: "de" # espeak-ng voice code used for phonemisation (de, en-us, …)
|
||||||
|
|
||||||
audio:
|
audio:
|
||||||
|
|||||||
+6
-3
@@ -494,9 +494,12 @@ The overnight scheduler runs `run_pipeline()` via `asyncio.to_thread`; the threa
|
|||||||
on the training subprocess for hours. Cancelling the scheduler task (quit) released
|
on the training subprocess for hours. Cancelling the scheduler task (quit) released
|
||||||
the training lock immediately while the thread kept training, and at interpreter exit
|
the training lock immediately while the thread kept training, and at interpreter exit
|
||||||
Python joins executor threads — so Javis hung on quit until training finished.
|
Python joins executor threads — so Javis hung on quit until training finished.
|
||||||
**Fix:** cancellation calls `piper_train.request_stop()`, which terminates the child
|
**Fix:** cancellation calls `piper_train.request_stop()`: the training process finishes
|
||||||
process (`_run` now polls instead of blocking in `wait()`); the lock is only released
|
its step, saves its progress — within the epoch too: the next run continues with the
|
||||||
once the worker has actually finished.
|
next batch — and exits (it is killed if that takes longer than
|
||||||
|
`STOP_GRACE_SECONDS`), and the scheduler waits for that (`_run` polls instead of
|
||||||
|
blocking in `wait()`). The lock (`training/.training.lock`, held by `run_pipeline`)
|
||||||
|
names the training processes and is only released once they have exited.
|
||||||
|
|
||||||
#### VOI-01
|
#### VOI-01
|
||||||
**Imported voice archives may carry pickle checkpoints** — *Medium (trust)*
|
**Imported voice archives may carry pickle checkpoints** — *Medium (trust)*
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ screen (GUI mode) and played back through the device's speakers.
|
|||||||
If the master's turn fails it returns its spoken error phrase instead, so a
|
If the master's turn fails it returns its spoken error phrase instead, so a
|
||||||
screenless satellite still gets audible feedback; connection problems play
|
screenless satellite still gets audible feedback; connection problems play
|
||||||
a low falling tone.
|
a low falling tone.
|
||||||
|
5. **Conversation view** — The GUI lists the conversation with the newest
|
||||||
|
message on top; scroll down for older ones. The fullscreen button in the
|
||||||
|
conversation card (or `F11`) toggles window fullscreen; the orb and the
|
||||||
|
conversation stay visible. `Esc` leaves fullscreen.
|
||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
|
|||||||
@@ -132,6 +132,7 @@ QPushButton[variant="danger"] {{ color: {CL_ERROR_FG}; }}
|
|||||||
QPushButton[variant="danger"]:hover {{ background: {CL_ERROR_BG}; border-color: {CL_ERROR_BORDER}; }}
|
QPushButton[variant="danger"]:hover {{ background: {CL_ERROR_BG}; border-color: {CL_ERROR_BORDER}; }}
|
||||||
QPushButton[variant="icon"] {{ padding: 0; min-height: 0; border-radius: 16px; }}
|
QPushButton[variant="icon"] {{ padding: 0; min-height: 0; border-radius: 16px; }}
|
||||||
QPushButton[variant="icon"]:checked {{ background: {CL_ERROR}; border-color: {CL_ERROR}; }}
|
QPushButton[variant="icon"]:checked {{ background: {CL_ERROR}; border-color: {CL_ERROR}; }}
|
||||||
|
QPushButton[variant="icon"][small="true"] {{ min-width: 46px; min-height: 46px; }}
|
||||||
|
|
||||||
QScrollArea {{ background: transparent; border: 0; }}
|
QScrollArea {{ background: transparent; border: 0; }}
|
||||||
QScrollBar:vertical {{ background: transparent; width: 8px; margin: 4px 0; }}
|
QScrollBar:vertical {{ background: transparent; width: 8px; margin: 4px 0; }}
|
||||||
@@ -211,9 +212,28 @@ def _draw_tune(p: QtGui.QPainter, box: QtCore.QRectF, color: QtGui.QColor) -> No
|
|||||||
p.drawEllipse(QtCore.QPointF(x + knob * s, y + row * s), 0.09 * s, 0.09 * s)
|
p.drawEllipse(QtCore.QPointF(x + knob * s, y + row * s), 0.09 * s, 0.09 * s)
|
||||||
|
|
||||||
|
|
||||||
|
def _draw_expand(p: QtGui.QPainter, box: QtCore.QRectF, color: QtGui.QColor, collapse: bool) -> None:
|
||||||
|
"""Four corner brackets: pointing outwards to enter, inwards to leave fullscreen."""
|
||||||
|
s, x, y = box.width(), box.x(), box.y()
|
||||||
|
pen = QtGui.QPen(color, max(1.5, s * 0.075))
|
||||||
|
pen.setCapStyle(QtCore.Qt.RoundCap)
|
||||||
|
pen.setJoinStyle(QtCore.Qt.RoundJoin)
|
||||||
|
p.setPen(pen)
|
||||||
|
p.setBrush(QtCore.Qt.NoBrush)
|
||||||
|
leg = 0.26 * s
|
||||||
|
reach = 0.10 if collapse else 0.36 # corner distance from the centre
|
||||||
|
d = 1 if collapse else -1 # legs run away from / towards the centre
|
||||||
|
for sx in (-1, 1):
|
||||||
|
for sy in (-1, 1):
|
||||||
|
vx, vy = x + (0.5 + sx * reach) * s, y + (0.5 + sy * reach) * s
|
||||||
|
p.drawPolyline([QtCore.QPointF(vx + d * sx * leg, vy),
|
||||||
|
QtCore.QPointF(vx, vy),
|
||||||
|
QtCore.QPointF(vx, vy + d * sy * leg)])
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=None)
|
@lru_cache(maxsize=None)
|
||||||
def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon:
|
def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon:
|
||||||
"""Render a crisp (2x) icon: 'mic', 'mic_off' or 'tune'."""
|
"""Render a crisp (2x) icon: 'mic', 'mic_off', 'tune', 'expand' or 'collapse'."""
|
||||||
pm = QtGui.QPixmap(size * 2, size * 2)
|
pm = QtGui.QPixmap(size * 2, size * 2)
|
||||||
pm.setDevicePixelRatio(2.0)
|
pm.setDevicePixelRatio(2.0)
|
||||||
pm.fill(QtCore.Qt.transparent)
|
pm.fill(QtCore.Qt.transparent)
|
||||||
@@ -222,6 +242,8 @@ def _icon(kind: str, color: str, size: int = 28) -> QtGui.QIcon:
|
|||||||
box = QtCore.QRectF(0, 0, size, size)
|
box = QtCore.QRectF(0, 0, size, size)
|
||||||
if kind == "tune":
|
if kind == "tune":
|
||||||
_draw_tune(p, box, QtGui.QColor(color))
|
_draw_tune(p, box, QtGui.QColor(color))
|
||||||
|
elif kind in ("expand", "collapse"):
|
||||||
|
_draw_expand(p, box, QtGui.QColor(color), collapse=(kind == "collapse"))
|
||||||
else:
|
else:
|
||||||
_draw_mic(p, box, QtGui.QColor(color), slashed=(kind == "mic_off"))
|
_draw_mic(p, box, QtGui.QColor(color), slashed=(kind == "mic_off"))
|
||||||
p.end()
|
p.end()
|
||||||
@@ -574,15 +596,18 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
Alexa-style UX features:
|
Alexa-style UX features:
|
||||||
- Mute toggle button on the orb
|
- Mute toggle button on the orb
|
||||||
- Connection health monitoring with retry countdown
|
- Connection health monitoring with retry countdown
|
||||||
- Transcript history (scrollable conversation log)
|
- Transcript history (scrollable conversation log, newest message on top)
|
||||||
|
- Fullscreen button in the conversation card (orb and chat stay visible)
|
||||||
- Visual mute indicator overlay
|
- Visual mute indicator overlay
|
||||||
"""
|
"""
|
||||||
|
|
||||||
settingsRequested = QtCore.Signal()
|
settingsRequested = QtCore.Signal()
|
||||||
muteToggled = QtCore.Signal(bool) # new signal for mute toggle
|
muteToggled = QtCore.Signal(bool) # new signal for mute toggle
|
||||||
|
fullscreenToggleRequested = QtCore.Signal()
|
||||||
|
|
||||||
def __init__(self, parent=None):
|
def __init__(self, parent=None):
|
||||||
super().__init__(parent)
|
super().__init__(parent)
|
||||||
|
self._fullscreen = False
|
||||||
self.setFocusPolicy(QtCore.Qt.ClickFocus) # tapping the background clears focus
|
self.setFocusPolicy(QtCore.Qt.ClickFocus) # tapping the background clears focus
|
||||||
outer = QtWidgets.QVBoxLayout(self)
|
outer = QtWidgets.QVBoxLayout(self)
|
||||||
outer.setContentsMargins(28, 20, 28, 24)
|
outer.setContentsMargins(28, 20, 28, 24)
|
||||||
@@ -654,10 +679,24 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
cv.setContentsMargins(24, 18, 14, 18)
|
cv.setContentsMargins(24, 18, 14, 18)
|
||||||
cv.setSpacing(10)
|
cv.setSpacing(10)
|
||||||
|
|
||||||
# Header label
|
# Header row: caption + fullscreen-chat toggle
|
||||||
|
header_row = QtWidgets.QHBoxLayout()
|
||||||
|
header_row.setContentsMargins(0, 0, 0, 0)
|
||||||
history_header = QtWidgets.QLabel("Konversation")
|
history_header = QtWidgets.QLabel("Konversation")
|
||||||
history_header.setProperty("role", "caption")
|
history_header.setProperty("role", "caption")
|
||||||
cv.addWidget(history_header)
|
header_row.addWidget(history_header)
|
||||||
|
header_row.addStretch(1)
|
||||||
|
self.expand_btn = QtWidgets.QPushButton()
|
||||||
|
self.expand_btn.setProperty("variant", "icon")
|
||||||
|
self.expand_btn.setProperty("small", True) # 48 px: the stylesheet's min-height would shrink it
|
||||||
|
self.expand_btn.setFixedSize(48, 48)
|
||||||
|
self.expand_btn.setIconSize(QtCore.QSize(26, 26))
|
||||||
|
self.expand_btn.setCursor(QtCore.Qt.PointingHandCursor)
|
||||||
|
self.expand_btn.setFocusPolicy(QtCore.Qt.TabFocus)
|
||||||
|
self.expand_btn.clicked.connect(self.fullscreenToggleRequested.emit)
|
||||||
|
self._sync_expand_button()
|
||||||
|
header_row.addWidget(self.expand_btn)
|
||||||
|
cv.addLayout(header_row)
|
||||||
|
|
||||||
# Transcript history (replaces single-turn display).
|
# Transcript history (replaces single-turn display).
|
||||||
# NOTE: this layout must belong to *exactly one* parent — we set it on
|
# NOTE: this layout must belong to *exactly one* parent — we set it on
|
||||||
@@ -708,6 +747,20 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
self.mute_btn.setToolTip(tip)
|
self.mute_btn.setToolTip(tip)
|
||||||
self.mute_btn.setAccessibleName(tip)
|
self.mute_btn.setAccessibleName(tip)
|
||||||
|
|
||||||
|
def set_fullscreen(self, fullscreen: bool) -> None:
|
||||||
|
"""Mirror the window's fullscreen state in the toggle button."""
|
||||||
|
if fullscreen == self._fullscreen:
|
||||||
|
return
|
||||||
|
self._fullscreen = fullscreen
|
||||||
|
self._sync_expand_button()
|
||||||
|
|
||||||
|
def _sync_expand_button(self) -> None:
|
||||||
|
fs = self._fullscreen
|
||||||
|
self.expand_btn.setIcon(_icon("collapse" if fs else "expand", CL_TEXT, 26))
|
||||||
|
tip = "Vollbild beenden" if fs else "Vollbild"
|
||||||
|
self.expand_btn.setToolTip(tip)
|
||||||
|
self.expand_btn.setAccessibleName(tip)
|
||||||
|
|
||||||
def _bubble_max_width(self) -> int:
|
def _bubble_max_width(self) -> int:
|
||||||
return max(240, int(self.transcript_scroll.viewport().width() * 0.8))
|
return max(240, int(self.transcript_scroll.viewport().width() * 0.8))
|
||||||
|
|
||||||
@@ -767,7 +820,8 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
else:
|
else:
|
||||||
h.addWidget(bubble)
|
h.addWidget(bubble)
|
||||||
h.addStretch(1)
|
h.addStretch(1)
|
||||||
self.transcript_layout.addWidget(row)
|
# Newest first: the latest bubble goes above the older ones.
|
||||||
|
self.transcript_layout.insertWidget(0, row)
|
||||||
|
|
||||||
# ── slots (called via Qt signals from the engine bridge) ──────────────
|
# ── slots (called via Qt signals from the engine bridge) ──────────────
|
||||||
|
|
||||||
@@ -796,9 +850,8 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
def _ensure_transcript_spacer(self) -> None:
|
def _ensure_transcript_spacer(self) -> None:
|
||||||
"""Keep exactly one expanding spacer, at the end of the transcript.
|
"""Keep exactly one expanding spacer, at the end of the transcript.
|
||||||
|
|
||||||
Bubbles are appended after the previous spacer, so stale spacers must
|
It pushes short transcripts to the top; stray spacers (e.g. left from
|
||||||
be removed from anywhere in the layout (they used to pile up between
|
the empty state) would open gaps between bubbles.
|
||||||
turns and open large gaps).
|
|
||||||
"""
|
"""
|
||||||
for i in reversed(range(self.transcript_layout.count())):
|
for i in reversed(range(self.transcript_layout.count())):
|
||||||
item = self.transcript_layout.itemAt(i)
|
item = self.transcript_layout.itemAt(i)
|
||||||
@@ -809,16 +862,16 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
self.transcript_layout.addSpacerItem(spacer)
|
self.transcript_layout.addSpacerItem(spacer)
|
||||||
|
|
||||||
def _prune_transcript(self, max_bubbles: int = 40) -> None:
|
def _prune_transcript(self, max_bubbles: int = 40) -> None:
|
||||||
"""Keep at most *max_bubbles* bubbles in the transcript layout (oldest first)."""
|
"""Keep at most *max_bubbles* bubbles in the transcript layout (drops the oldest, at the bottom)."""
|
||||||
rows = [self.transcript_layout.itemAt(i).widget()
|
rows = [self.transcript_layout.itemAt(i).widget()
|
||||||
for i in range(self.transcript_layout.count())]
|
for i in range(self.transcript_layout.count())]
|
||||||
rows = [row for row in rows if row is not None]
|
rows = [row for row in rows if row is not None]
|
||||||
for row in rows[:max(0, len(rows) - max_bubbles)]:
|
for row in rows[max_bubbles:]:
|
||||||
self.transcript_layout.removeWidget(row)
|
self.transcript_layout.removeWidget(row)
|
||||||
row.deleteLater()
|
row.deleteLater()
|
||||||
|
|
||||||
def set_turn(self, user_text: str, reply_text: str) -> None:
|
def set_turn(self, user_text: str, reply_text: str) -> None:
|
||||||
"""Add a new conversation turn to the transcript history (Alexa-style)."""
|
"""Add a new conversation turn to the transcript history (newest on top)."""
|
||||||
self._add_bubble(user_text, "user")
|
self._add_bubble(user_text, "user")
|
||||||
|
|
||||||
# Assistant message bubble (if any)
|
# Assistant message bubble (if any)
|
||||||
@@ -826,16 +879,16 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
self._add_bubble(reply_text, "assistant")
|
self._add_bubble(reply_text, "assistant")
|
||||||
|
|
||||||
self._prune_transcript()
|
self._prune_transcript()
|
||||||
# Spacer to push content to bottom (like Alexa's conversation history)
|
# Keeps short transcripts hugging the top edge
|
||||||
self._ensure_transcript_spacer()
|
self._ensure_transcript_spacer()
|
||||||
|
|
||||||
# Auto-scroll to bottom
|
# Auto-scroll to top
|
||||||
QTimer.singleShot(50, self._scroll_transcript_to_bottom)
|
QTimer.singleShot(50, self._scroll_transcript_to_top)
|
||||||
|
|
||||||
def _scroll_transcript_to_bottom(self) -> None:
|
def _scroll_transcript_to_top(self) -> None:
|
||||||
"""Auto-scroll transcript to show latest messages."""
|
"""Auto-scroll transcript to show the latest messages (newest is on top)."""
|
||||||
vbar = self.transcript_scroll.verticalScrollBar()
|
vbar = self.transcript_scroll.verticalScrollBar()
|
||||||
vbar.setValue(vbar.maximum())
|
vbar.setValue(vbar.minimum())
|
||||||
|
|
||||||
def set_connection(self, ok: bool, retry_count: int = 0, backoff: float = 0.0) -> None:
|
def set_connection(self, ok: bool, retry_count: int = 0, backoff: float = 0.0) -> None:
|
||||||
if ok:
|
if ok:
|
||||||
@@ -865,7 +918,7 @@ class MainScreen(QtWidgets.QWidget):
|
|||||||
self._add_bubble(f"⚠ {message}", "error")
|
self._add_bubble(f"⚠ {message}", "error")
|
||||||
self._prune_transcript()
|
self._prune_transcript()
|
||||||
self._ensure_transcript_spacer()
|
self._ensure_transcript_spacer()
|
||||||
QTimer.singleShot(50, self._scroll_transcript_to_bottom)
|
QTimer.singleShot(50, self._scroll_transcript_to_top)
|
||||||
|
|
||||||
|
|
||||||
class SlaveWindow(QtWidgets.QMainWindow):
|
class SlaveWindow(QtWidgets.QMainWindow):
|
||||||
@@ -907,6 +960,7 @@ class SlaveWindow(QtWidgets.QMainWindow):
|
|||||||
self.engineError.connect(self.main_screen.show_error)
|
self.engineError.connect(self.main_screen.show_error)
|
||||||
self.engineConnection.connect(self.main_screen.set_connection)
|
self.engineConnection.connect(self.main_screen.set_connection)
|
||||||
self.engineRoom.connect(lambda r: self.main_screen.set_room(r))
|
self.engineRoom.connect(lambda r: self.main_screen.set_room(r))
|
||||||
|
self.main_screen.fullscreenToggleRequested.connect(self._toggle_fullscreen)
|
||||||
|
|
||||||
QtGui.QShortcut(QtGui.QKeySequence("F11"), self, activated=self._toggle_fullscreen)
|
QtGui.QShortcut(QtGui.QKeySequence("F11"), self, activated=self._toggle_fullscreen)
|
||||||
# Kiosk: fullscreen with a hidden cursor (touch panels); Esc leaves fullscreen.
|
# Kiosk: fullscreen with a hidden cursor (touch panels); Esc leaves fullscreen.
|
||||||
@@ -919,6 +973,12 @@ class SlaveWindow(QtWidgets.QMainWindow):
|
|||||||
self.showFullScreen()
|
self.showFullScreen()
|
||||||
QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CursorShape.BlankCursor)
|
QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CursorShape.BlankCursor)
|
||||||
|
|
||||||
|
def changeEvent(self, event): # noqa: N802
|
||||||
|
super().changeEvent(event)
|
||||||
|
if event.type() == QtCore.QEvent.Type.WindowStateChange:
|
||||||
|
# Covers every path in (button, F11, kiosk, settings) and out (Esc).
|
||||||
|
self.main_screen.set_fullscreen(self.isFullScreen())
|
||||||
|
|
||||||
def _leave_fullscreen(self) -> None:
|
def _leave_fullscreen(self) -> None:
|
||||||
if self.isFullScreen():
|
if self.isFullScreen():
|
||||||
self.showNormal()
|
self.showNormal()
|
||||||
|
|||||||
+358
-41
@@ -2,9 +2,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import contextlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
@@ -16,9 +18,9 @@ from ..audio_io.vad import make_vad
|
|||||||
from ..audio_io.wake_word import WakeWordDetector
|
from ..audio_io.wake_word import WakeWordDetector
|
||||||
from ..core.config import Config
|
from ..core.config import Config
|
||||||
from ..core.events import EventBus
|
from ..core.events import EventBus
|
||||||
|
from ..core.hardware import GpuUnavailableError
|
||||||
from ..core.logger import get_logger
|
from ..core.logger import get_logger
|
||||||
from ..core.state import AgentState
|
from ..core.state import AgentState
|
||||||
from ..core.hardware import GpuUnavailableError
|
|
||||||
from ..core.tasks import cancel_and_wait, spawn
|
from ..core.tasks import cancel_and_wait, spawn
|
||||||
from ..llm.conversation import Conversation
|
from ..llm.conversation import Conversation
|
||||||
from ..llm.hermes_client import HermesClient
|
from ..llm.hermes_client import HermesClient
|
||||||
@@ -48,16 +50,17 @@ from .middleware import (
|
|||||||
drain_background_tasks,
|
drain_background_tasks,
|
||||||
)
|
)
|
||||||
from .prompts import load_system_prompt
|
from .prompts import load_system_prompt
|
||||||
|
from ..stt.unidentified import UnidentifiedStore
|
||||||
from ..stt.voice_profiler import NullProfiler, SpeakerPrivileges, VoiceProfiler
|
from ..stt.voice_profiler import NullProfiler, SpeakerPrivileges, VoiceProfiler
|
||||||
|
|
||||||
log = get_logger("agent")
|
log = get_logger("agent")
|
||||||
|
|
||||||
SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$")
|
|
||||||
|
|
||||||
# Detects whether the last sentence of a reply is a question.
|
|
||||||
# Returned by Orchestrator._identify_speaker when speaker ID raised.
|
# Returned by Orchestrator._identify_speaker when speaker ID raised.
|
||||||
_ID_FAILED = object()
|
_ID_FAILED = object()
|
||||||
|
|
||||||
|
SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$")
|
||||||
|
|
||||||
|
# Detects whether the last sentence of a reply is a question.
|
||||||
_FOLLOWUP_RE = re.compile(r'\?\s*$')
|
_FOLLOWUP_RE = re.compile(r'\?\s*$')
|
||||||
|
|
||||||
|
|
||||||
@@ -353,6 +356,37 @@ class RemoteTurnResult:
|
|||||||
expects_followup: bool = False
|
expects_followup: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class VoiceSample:
|
||||||
|
"""A live voice sample recorded from the GUI to enhance a profile."""
|
||||||
|
audio: np.ndarray
|
||||||
|
sample_rate: int
|
||||||
|
embedding: Optional[np.ndarray] = None
|
||||||
|
# Cosine similarity to the target profile (None if it does not exist yet).
|
||||||
|
score: Optional[float] = None
|
||||||
|
best_name: Optional[str] = None
|
||||||
|
best_score: Optional[float] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def duration_s(self) -> float:
|
||||||
|
return len(self.audio) / float(self.sample_rate) if self.sample_rate else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _PendingEnrollClip:
|
||||||
|
"""Unknown-speaker clip buffered while the "who is this?" prompt is open."""
|
||||||
|
audio: np.ndarray
|
||||||
|
sample_rate: int
|
||||||
|
# Task storing the same clip in the unidentified inbox (returns its id).
|
||||||
|
store_task: Optional["asyncio.Task[Optional[str]]"] = None
|
||||||
|
|
||||||
|
def inbox_id(self) -> Optional[str]:
|
||||||
|
t = self.store_task
|
||||||
|
if t is None or not t.done() or t.cancelled() or t.exception() is not None:
|
||||||
|
return None
|
||||||
|
return t.result()
|
||||||
|
|
||||||
|
|
||||||
class Orchestrator:
|
class Orchestrator:
|
||||||
"""Long-lived voice agent. Owns all subsystems."""
|
"""Long-lived voice agent. Owns all subsystems."""
|
||||||
|
|
||||||
@@ -441,8 +475,16 @@ class Orchestrator:
|
|||||||
self.profiler: VoiceProfiler | NullProfiler = (
|
self.profiler: VoiceProfiler | NullProfiler = (
|
||||||
VoiceProfiler(config) if config.speaker_id.enabled else NullProfiler()
|
VoiceProfiler(config) if config.speaker_id.enabled else NullProfiler()
|
||||||
)
|
)
|
||||||
self._enroll_clip_buffer: list[np.ndarray] = []
|
self.unidentified = UnidentifiedStore(
|
||||||
|
config.paths.voice_profiles_dir / "unidentified",
|
||||||
|
config.speaker_id.max_unidentified,
|
||||||
|
)
|
||||||
|
self._enroll_clip_buffer: list[_PendingEnrollClip] = []
|
||||||
self._enroll_ask_pending: bool = False
|
self._enroll_ask_pending: bool = False
|
||||||
|
# Cleared while the GUI records voice samples so the wake-word loop
|
||||||
|
# (which reads the mic outside _turn_lock) stops competing for chunks.
|
||||||
|
self._mic_free = asyncio.Event()
|
||||||
|
self._mic_free.set()
|
||||||
self._bg_quick_turn_count: int = 0
|
self._bg_quick_turn_count: int = 0
|
||||||
self._tool_cache: dict[str, tuple[str, float]] = {}
|
self._tool_cache: dict[str, tuple[str, float]] = {}
|
||||||
|
|
||||||
@@ -560,8 +602,13 @@ class Orchestrator:
|
|||||||
# Disable any backend whose load() raised so later synthesis/prewarm
|
# Disable any backend whose load() raised so later synthesis/prewarm
|
||||||
# calls don't encounter RuntimeError("load() has not been called").
|
# calls don't encounter RuntimeError("load() has not been called").
|
||||||
if self.tts is not None and getattr(self.tts, "_voice", True) is None:
|
if self.tts is not None and getattr(self.tts, "_voice", True) is None:
|
||||||
log.warning("TTS model failed to load — speech output disabled")
|
backend, self.tts = self.tts, None
|
||||||
self.tts = None
|
if hasattr(backend, "model_available"):
|
||||||
|
log.warning("TTS model failed to load — speech output disabled until "
|
||||||
|
"a model is available")
|
||||||
|
spawn(self._revive_tts(backend), name="tts-revive")
|
||||||
|
else:
|
||||||
|
log.warning("TTS model failed to load — speech output disabled")
|
||||||
await self.bus.publish("mcp_ready", self.mcp.status_snapshot())
|
await self.bus.publish("mcp_ready", self.mcp.status_snapshot())
|
||||||
ok = await self.llm.health()
|
ok = await self.llm.health()
|
||||||
await self.bus.publish("llm_health", {"ok": ok, "url": self.config.llm.base_url, "model": self.config.llm.model})
|
await self.bus.publish("llm_health", {"ok": ok, "url": self.config.llm.base_url, "model": self.config.llm.model})
|
||||||
@@ -854,8 +901,12 @@ class Orchestrator:
|
|||||||
await self.bus.publish(
|
await self.bus.publish(
|
||||||
"speaker_id", {"name": None, "confidence": 0.0, "known": False}
|
"speaker_id", {"name": None, "confidence": 0.0, "known": False}
|
||||||
)
|
)
|
||||||
if self.config.speaker_id.auto_enroll:
|
source = device_name or "satellite"
|
||||||
await self._auto_enroll_speaker(audio)
|
if room:
|
||||||
|
source = f"{source} ({room})"
|
||||||
|
await self._handle_unknown_speaker(
|
||||||
|
audio, sample_rate, text, source=source,
|
||||||
|
)
|
||||||
|
|
||||||
if not _privileges.can_respond:
|
if not _privileges.can_respond:
|
||||||
log.info(
|
log.info(
|
||||||
@@ -947,6 +998,10 @@ class Orchestrator:
|
|||||||
False when the listen loop exited without a detection (stale timeout,
|
False when the listen loop exited without a detection (stale timeout,
|
||||||
``_stop`` set, or mic disconnected)."""
|
``_stop`` set, or mic disconnected)."""
|
||||||
assert self.wake is not None
|
assert self.wake is not None
|
||||||
|
# A GUI voice-sample recording owns the mic — wait until it is done.
|
||||||
|
mic_free: Optional[asyncio.Event] = getattr(self, "_mic_free", None)
|
||||||
|
if mic_free is not None and not mic_free.is_set():
|
||||||
|
await mic_free.wait()
|
||||||
await self.set_state(AgentState.IDLE)
|
await self.set_state(AgentState.IDLE)
|
||||||
self.wake.reset()
|
self.wake.reset()
|
||||||
if self.mic is not None:
|
if self.mic is not None:
|
||||||
@@ -954,6 +1009,8 @@ class Orchestrator:
|
|||||||
log.info(f"listening for wake word: '{self.config.wake_word.model}'")
|
log.info(f"listening for wake word: '{self.config.wake_word.model}'")
|
||||||
last_pub = 0.0
|
last_pub = 0.0
|
||||||
async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S):
|
async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S):
|
||||||
|
if mic_free is not None and not mic_free.is_set():
|
||||||
|
return False # handed over to an exclusive recording
|
||||||
score = self.wake.feed(chunk)
|
score = self.wake.feed(chunk)
|
||||||
# Everything heard before the wake word is background (music, TV,
|
# Everything heard before the wake word is background (music, TV,
|
||||||
# this device's own speakers), so the recorder's voice gate is
|
# this device's own speakers), so the recorder's voice gate is
|
||||||
@@ -1213,6 +1270,14 @@ class Orchestrator:
|
|||||||
return True
|
return True
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any:
|
||||||
|
"""Run speaker ID off-loop; a failure must never abort the turn."""
|
||||||
|
try:
|
||||||
|
return await asyncio.to_thread(self.profiler.identify, audio, sample_rate)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.exception(f"[profiler] speaker identification failed: {exc}")
|
||||||
|
return _ID_FAILED
|
||||||
|
|
||||||
def _unknown_privileges(self) -> SpeakerPrivileges:
|
def _unknown_privileges(self) -> SpeakerPrivileges:
|
||||||
"""Build a :class:`SpeakerPrivileges` from the unknown-speaker config fields."""
|
"""Build a :class:`SpeakerPrivileges` from the unknown-speaker config fields."""
|
||||||
cfg = self.config.speaker_id
|
cfg = self.config.speaker_id
|
||||||
@@ -1228,6 +1293,32 @@ class Orchestrator:
|
|||||||
denied_tools=denied,
|
denied_tools=denied,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def _revive_tts(self, backend, interval: float = 30.0) -> None:
|
||||||
|
"""Turn speech output on once the TTS model that failed to load is there.
|
||||||
|
|
||||||
|
E.g. a voice still in its first training run: training exports the
|
||||||
|
voice's model as it goes (see ``javis.voices.piper_train``).
|
||||||
|
"""
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(self._stop.wait(), timeout=interval)
|
||||||
|
return
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
if not backend.model_available():
|
||||||
|
continue
|
||||||
|
await asyncio.to_thread(backend.load)
|
||||||
|
except Exception as exc: # noqa: BLE001 - tried again once the model changes
|
||||||
|
log.warning(f"TTS model still failed to load: {exc}")
|
||||||
|
continue
|
||||||
|
if self._stop.is_set():
|
||||||
|
return
|
||||||
|
self.tts = backend
|
||||||
|
log.info("TTS model loaded — speech output enabled")
|
||||||
|
spawn(self._prewarm_tts(), name="tts-prewarm")
|
||||||
|
return
|
||||||
|
|
||||||
async def _prewarm_tts(self) -> None:
|
async def _prewarm_tts(self) -> None:
|
||||||
"""Silently synthesize a short phrase to warm up TTS model buffers.
|
"""Silently synthesize a short phrase to warm up TTS model buffers.
|
||||||
|
|
||||||
@@ -1270,14 +1361,6 @@ class Orchestrator:
|
|||||||
# Reset to IDLE so the GUI doesn't stay stuck on "listening"
|
# Reset to IDLE so the GUI doesn't stay stuck on "listening"
|
||||||
# when the turn ends without any speech.
|
# when the turn ends without any speech.
|
||||||
await self.set_state(AgentState.IDLE)
|
await self.set_state(AgentState.IDLE)
|
||||||
async def _identify_speaker(self, audio: np.ndarray, sample_rate: int) -> Any:
|
|
||||||
"""Run speaker ID off-loop; a failure must never abort the turn."""
|
|
||||||
try:
|
|
||||||
return await asyncio.to_thread(self.profiler.identify, audio, sample_rate)
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
log.exception(f"[profiler] speaker identification failed: {exc}")
|
|
||||||
return _ID_FAILED
|
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|
||||||
await self.set_state(AgentState.THINKING)
|
await self.set_state(AgentState.THINKING)
|
||||||
@@ -1347,8 +1430,9 @@ class Orchestrator:
|
|||||||
await self.bus.publish(
|
await self.bus.publish(
|
||||||
"speaker_id", {"name": None, "confidence": 0.0, "known": False}
|
"speaker_id", {"name": None, "confidence": 0.0, "known": False}
|
||||||
)
|
)
|
||||||
if self.config.speaker_id.auto_enroll:
|
await self._handle_unknown_speaker(
|
||||||
await self._auto_enroll_speaker(audio)
|
audio, self.config.audio.sample_rate, tr.text or "", source="local",
|
||||||
|
)
|
||||||
|
|
||||||
# ── privilege: can_respond check ──────────────────────────────────────
|
# ── privilege: can_respond check ──────────────────────────────────────
|
||||||
if not _privileges.can_respond:
|
if not _privileges.can_respond:
|
||||||
@@ -2007,8 +2091,7 @@ class Orchestrator:
|
|||||||
|
|
||||||
n = self.config.speaker_id.min_enroll_clips
|
n = self.config.speaker_id.min_enroll_clips
|
||||||
clips: list[np.ndarray] = []
|
clips: list[np.ndarray] = []
|
||||||
async with self._turn_lock:
|
async with self._exclusive_mic():
|
||||||
await self.set_state(AgentState.ENROLLING)
|
|
||||||
for i in range(n):
|
for i in range(n):
|
||||||
await self.bus.publish("enroll_progress", {
|
await self.bus.publish("enroll_progress", {
|
||||||
"name": name, "clip": i, "total": n, "recording": True,
|
"name": name, "clip": i, "total": n, "recording": True,
|
||||||
@@ -2016,7 +2099,6 @@ class Orchestrator:
|
|||||||
spawn(self._play_wake_chime())
|
spawn(self._play_wake_chime())
|
||||||
audio = await record_utterance(self.config, self.mic, self.vad)
|
audio = await record_utterance(self.config, self.mic, self.vad)
|
||||||
if audio is None or len(audio) < 1600:
|
if audio is None or len(audio) < 1600:
|
||||||
await self.set_state(AgentState.IDLE)
|
|
||||||
await self.bus.publish("enroll_progress", {
|
await self.bus.publish("enroll_progress", {
|
||||||
"name": name, "clip": i, "total": n,
|
"name": name, "clip": i, "total": n,
|
||||||
"error": "Kein Audio erkannt — Aufnahme abgebrochen.",
|
"error": "Kein Audio erkannt — Aufnahme abgebrochen.",
|
||||||
@@ -2030,48 +2112,283 @@ class Orchestrator:
|
|||||||
sr = self.config.audio.sample_rate
|
sr = self.config.audio.sample_rate
|
||||||
await asyncio.to_thread(self.profiler.enroll, name, clips, sr)
|
await asyncio.to_thread(self.profiler.enroll, name, clips, sr)
|
||||||
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)})
|
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)})
|
||||||
await self.set_state(AgentState.IDLE)
|
|
||||||
log.info(f"[profiler] enrollment complete for '{name}'")
|
log.info(f"[profiler] enrollment complete for '{name}'")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def _exclusive_mic(self, wait_s: float = 0.0) -> AsyncIterator[MicStream]:
|
||||||
|
"""Own the microphone for a GUI-initiated recording.
|
||||||
|
|
||||||
|
Takes ``_turn_lock`` (waiting up to *wait_s*) and pauses the wake-word
|
||||||
|
loop via ``_mic_free`` so every chunk reaches the recorder. Raises
|
||||||
|
``RuntimeError`` with a user-facing message when that is not possible.
|
||||||
|
"""
|
||||||
|
mic = self.mic
|
||||||
|
if mic is None:
|
||||||
|
raise RuntimeError("No microphone available — voice input is disabled.")
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(self._turn_lock.acquire(), timeout=max(0.01, wait_s))
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise RuntimeError("Javis is busy right now — try again in a moment.") from None
|
||||||
|
self._mic_free.clear()
|
||||||
|
try:
|
||||||
|
await self.set_state(AgentState.ENROLLING)
|
||||||
|
mic.drain()
|
||||||
|
yield mic
|
||||||
|
finally:
|
||||||
|
self._mic_free.set()
|
||||||
|
try:
|
||||||
|
await self.set_state(AgentState.IDLE)
|
||||||
|
finally:
|
||||||
|
self._turn_lock.release()
|
||||||
|
|
||||||
|
def _require_profiler(self) -> None:
|
||||||
|
if not self.config.speaker_id.enabled:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Speaker identification is disabled (Settings → Speaker Identification).")
|
||||||
|
if not self.profiler.is_ready():
|
||||||
|
raise RuntimeError("The speaker identification model is not loaded yet.")
|
||||||
|
|
||||||
|
async def record_voice_sample(
|
||||||
|
self, name: Optional[str] = None, *, max_seconds: float = 15.0,
|
||||||
|
) -> VoiceSample:
|
||||||
|
"""Record one live utterance for enhancing *name*'s voice profile.
|
||||||
|
|
||||||
|
Nothing is stored — pass the result to :meth:`add_voice_samples` once
|
||||||
|
the user has reviewed it. ``score`` is the cosine similarity to
|
||||||
|
*name*'s current profile (``None`` if it has none). Raises
|
||||||
|
``RuntimeError`` with a user-facing message on failure.
|
||||||
|
"""
|
||||||
|
self._require_profiler()
|
||||||
|
sr = self.config.audio.sample_rate
|
||||||
|
async with self._exclusive_mic(wait_s=3.0) as mic:
|
||||||
|
# Chime first, then drop whatever the mic picked up of it.
|
||||||
|
await self._play_wake_chime()
|
||||||
|
mic.drain()
|
||||||
|
audio = await record_utterance(
|
||||||
|
self.config, mic, self.vad,
|
||||||
|
max_seconds=max_seconds, initial_grace_ms=5000,
|
||||||
|
)
|
||||||
|
if audio is None or len(audio) < int(sr * 0.5):
|
||||||
|
raise RuntimeError("No speech detected — please try again.")
|
||||||
|
emb = await asyncio.to_thread(self.profiler.embed, audio, sr)
|
||||||
|
ranked = self.profiler.rank(emb)
|
||||||
|
score = next((s for n, s in ranked if n == name), None) if name else None
|
||||||
|
best_name, best_score = ranked[0] if ranked else (None, None)
|
||||||
|
return VoiceSample(
|
||||||
|
audio=audio, sample_rate=sr, embedding=emb, score=score,
|
||||||
|
best_name=best_name, best_score=best_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def add_voice_samples(self, name: str, samples: list[VoiceSample]) -> int:
|
||||||
|
"""Blend recorded *samples* into *name*'s profile (created if missing).
|
||||||
|
|
||||||
|
Returns the number of samples used.
|
||||||
|
"""
|
||||||
|
name = (name or "").strip()
|
||||||
|
if not name:
|
||||||
|
raise ValueError("Profile name is empty.")
|
||||||
|
|
||||||
|
def _apply() -> tuple[int, bool]:
|
||||||
|
embs = []
|
||||||
|
for s in samples:
|
||||||
|
emb = s.embedding
|
||||||
|
if emb is None or not np.any(emb):
|
||||||
|
emb = self.profiler.embed(s.audio, s.sample_rate)
|
||||||
|
if np.any(emb):
|
||||||
|
embs.append(emb)
|
||||||
|
if not embs:
|
||||||
|
return 0, False
|
||||||
|
if self.profiler.has_profile(name):
|
||||||
|
used = sum(1 for e in embs if self.profiler.refine_embedding(name, e))
|
||||||
|
return used, False
|
||||||
|
return (len(embs), True) if self.profiler.enroll_embeddings(name, embs) else (0, False)
|
||||||
|
|
||||||
|
used, created = await asyncio.to_thread(_apply)
|
||||||
|
if not used:
|
||||||
|
raise RuntimeError("No usable voice samples — is the speaker model loaded?")
|
||||||
|
log.info(f"[profiler] added {used} live sample(s) to '{name}'")
|
||||||
|
await self.bus.publish("voice_profiles_changed", {
|
||||||
|
"name": name, "added": used, "created": created,
|
||||||
|
})
|
||||||
|
return used
|
||||||
|
|
||||||
|
# ---------------- unidentified speakers ----------------
|
||||||
|
|
||||||
|
async def _handle_unknown_speaker(
|
||||||
|
self, audio: np.ndarray, sample_rate: int, text: str, *, source: str,
|
||||||
|
) -> None:
|
||||||
|
"""Keep the clip in the unidentified inbox and/or ask for a name."""
|
||||||
|
store_task: Optional["asyncio.Task[Optional[str]]"] = None
|
||||||
|
if self.config.speaker_id.keep_unidentified and (text or "").strip():
|
||||||
|
store_task = spawn(
|
||||||
|
self._store_unidentified(audio, sample_rate, text, source),
|
||||||
|
name="unidentified-store",
|
||||||
|
)
|
||||||
|
if self.config.speaker_id.auto_enroll:
|
||||||
|
await self._auto_enroll_speaker(audio, sample_rate, store_task)
|
||||||
|
|
||||||
|
async def _store_unidentified(
|
||||||
|
self, audio: np.ndarray, sample_rate: int, text: str, source: str,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Embed *audio* and add it to the inbox. Returns the clip id."""
|
||||||
|
try:
|
||||||
|
emb = await asyncio.to_thread(self.profiler.embed, audio, sample_rate)
|
||||||
|
ranked = self.profiler.rank(emb)
|
||||||
|
best_name, best_score = ranked[0] if ranked else (None, None)
|
||||||
|
self.unidentified.max_clips = max(1, int(self.config.speaker_id.max_unidentified))
|
||||||
|
clip_id = await asyncio.to_thread(
|
||||||
|
self.unidentified.add, audio, sample_rate,
|
||||||
|
embedding=emb, text=text, source=source,
|
||||||
|
best_name=best_name, best_score=best_score,
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(f"[unidentified] could not store clip: {exc}")
|
||||||
|
return None
|
||||||
|
await self.bus.publish("unidentified_changed", {"added": clip_id})
|
||||||
|
return clip_id
|
||||||
|
|
||||||
|
async def assign_unidentified(self, clip_ids: list[str], name: str) -> dict:
|
||||||
|
"""Sort inbox clips into *name*'s profile, creating it if needed.
|
||||||
|
|
||||||
|
Returns ``{"name", "added", "created"}``. The clips are removed from
|
||||||
|
the inbox. Raises ``ValueError``/``RuntimeError`` with a user-facing
|
||||||
|
message on failure.
|
||||||
|
"""
|
||||||
|
name = (name or "").strip()
|
||||||
|
if not name:
|
||||||
|
raise ValueError("Profile name is empty.")
|
||||||
|
if not clip_ids:
|
||||||
|
raise ValueError("No clips selected.")
|
||||||
|
self._require_profiler()
|
||||||
|
|
||||||
|
def _apply() -> tuple[list[str], bool]:
|
||||||
|
used: list[str] = []
|
||||||
|
embs: list[np.ndarray] = []
|
||||||
|
for cid in clip_ids:
|
||||||
|
emb = self.unidentified.load_embedding(cid)
|
||||||
|
if emb is None:
|
||||||
|
try:
|
||||||
|
audio, sr = self.unidentified.load_audio(cid)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(f"[unidentified] clip {cid} unavailable: {exc}")
|
||||||
|
continue
|
||||||
|
emb = self.profiler.embed(audio, sr)
|
||||||
|
if np.any(emb):
|
||||||
|
embs.append(emb)
|
||||||
|
used.append(cid)
|
||||||
|
if not embs:
|
||||||
|
return [], False
|
||||||
|
if self.profiler.has_profile(name):
|
||||||
|
for e in embs:
|
||||||
|
self.profiler.refine_embedding(name, e)
|
||||||
|
created = False
|
||||||
|
else:
|
||||||
|
if not self.profiler.enroll_embeddings(name, embs):
|
||||||
|
return [], False
|
||||||
|
created = True
|
||||||
|
self.unidentified.remove(used)
|
||||||
|
return used, created
|
||||||
|
|
||||||
|
used, created = await asyncio.to_thread(_apply)
|
||||||
|
if not used:
|
||||||
|
raise RuntimeError("None of the selected clips could be used.")
|
||||||
|
# Don't let a still-open "who is this?" prompt enroll them a second time.
|
||||||
|
done = set(used)
|
||||||
|
self._enroll_clip_buffer = [
|
||||||
|
p for p in self._enroll_clip_buffer if p.inbox_id() not in done
|
||||||
|
]
|
||||||
|
log.info(
|
||||||
|
f"[profiler] sorted {len(used)} unidentified clip(s) into "
|
||||||
|
f"{'new' if created else 'existing'} profile '{name}'"
|
||||||
|
)
|
||||||
|
await self.bus.publish("voice_profiles_changed", {
|
||||||
|
"name": name, "added": len(used), "created": created,
|
||||||
|
})
|
||||||
|
await self.bus.publish("unidentified_changed", {"removed": used})
|
||||||
|
return {"name": name, "added": len(used), "created": created}
|
||||||
|
|
||||||
|
async def delete_unidentified(self, clip_ids: Optional[list[str]] = None) -> int:
|
||||||
|
"""Delete the given inbox clips, or all of them when *clip_ids* is None."""
|
||||||
|
if clip_ids is None:
|
||||||
|
removed = await asyncio.to_thread(self.unidentified.clear)
|
||||||
|
else:
|
||||||
|
removed = await asyncio.to_thread(self.unidentified.remove, clip_ids)
|
||||||
|
if removed:
|
||||||
|
await self.bus.publish("unidentified_changed", {"removed": clip_ids})
|
||||||
|
return removed
|
||||||
|
|
||||||
|
def pending_enroll_clips(self) -> list[tuple[np.ndarray, int]]:
|
||||||
|
"""Clips buffered for the open "who is this?" prompt (for playback)."""
|
||||||
|
return [(p.audio, p.sample_rate) for p in self._enroll_clip_buffer]
|
||||||
|
|
||||||
|
def auto_enroll_dismiss(self) -> None:
|
||||||
|
"""The user skipped the name prompt.
|
||||||
|
|
||||||
|
Drops the buffered clips (they stay in the unidentified inbox when
|
||||||
|
``keep_unidentified`` is on) and lets the prompt appear again later.
|
||||||
|
"""
|
||||||
|
self._enroll_clip_buffer.clear()
|
||||||
|
self._enroll_ask_pending = False
|
||||||
|
|
||||||
async def auto_enroll_confirm(self, key: str, name: str) -> bool:
|
async def auto_enroll_confirm(self, key: str, name: str) -> bool:
|
||||||
"""Finalize auto-enrollment using all clips buffered since the dialog opened.
|
"""Finalize auto-enrollment using all clips buffered since the dialog opened.
|
||||||
|
|
||||||
Called by the GUI when the user submits their name. *key* is kept for
|
Called by the GUI when the user submits their name. *key* is kept for
|
||||||
API compatibility but is no longer used to look up audio — the full
|
API compatibility but is no longer used to look up audio — the full
|
||||||
clip buffer is consumed instead.
|
clip buffer is consumed instead. Clips that were also stored in the
|
||||||
|
unidentified inbox are removed from it.
|
||||||
"""
|
"""
|
||||||
clips = list(self._enroll_clip_buffer)
|
pending = list(self._enroll_clip_buffer)
|
||||||
self._enroll_clip_buffer.clear()
|
self._enroll_clip_buffer.clear()
|
||||||
self._enroll_ask_pending = False
|
self._enroll_ask_pending = False
|
||||||
|
|
||||||
if not clips:
|
if not pending:
|
||||||
log.warning("[profiler] auto_enroll_confirm: clip buffer is empty")
|
log.warning("[profiler] auto_enroll_confirm: clip buffer is empty")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
sr = self.config.audio.sample_rate
|
def _apply() -> int:
|
||||||
existing_names = {p["name"] for p in self.profiler.list_profiles()}
|
embs = [self.profiler.embed(p.audio, p.sample_rate) for p in pending]
|
||||||
if name in existing_names:
|
if self.profiler.has_profile(name):
|
||||||
# Merge all buffered clips into the existing profile.
|
# Merge all buffered clips into the existing profile.
|
||||||
for clip in clips:
|
n = sum(1 for e in embs if self.profiler.refine_embedding(name, e))
|
||||||
await asyncio.to_thread(self.profiler.refine, name, clip, sr)
|
log.info(f"[profiler] merged {n} clip(s) into existing profile '{name}'")
|
||||||
log.info(
|
return n
|
||||||
f"[profiler] merged {len(clips)} clip(s) into existing profile '{name}'"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Build a new profile from all accumulated clips.
|
# Build a new profile from all accumulated clips.
|
||||||
await asyncio.to_thread(self.profiler.enroll, name, clips, sr)
|
return len(embs) if self.profiler.enroll_embeddings(name, embs) else 0
|
||||||
log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)")
|
|
||||||
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)})
|
used = await asyncio.to_thread(_apply)
|
||||||
|
if not used:
|
||||||
|
return False
|
||||||
|
|
||||||
|
inbox_ids: list[str] = []
|
||||||
|
for p in pending:
|
||||||
|
if p.store_task is not None:
|
||||||
|
try:
|
||||||
|
cid = await p.store_task
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
cid = None
|
||||||
|
if cid:
|
||||||
|
inbox_ids.append(cid)
|
||||||
|
if inbox_ids:
|
||||||
|
await self.delete_unidentified(inbox_ids)
|
||||||
|
await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": used})
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def _auto_enroll_speaker(self, audio: np.ndarray) -> None:
|
async def _auto_enroll_speaker(
|
||||||
|
self,
|
||||||
|
audio: np.ndarray,
|
||||||
|
sample_rate: Optional[int] = None,
|
||||||
|
store_task: Optional["asyncio.Task[Optional[str]]"] = None,
|
||||||
|
) -> None:
|
||||||
"""Buffer *audio* from an unknown speaker and ask for a name (once).
|
"""Buffer *audio* from an unknown speaker and ask for a name (once).
|
||||||
|
|
||||||
Subsequent unknown utterances while the dialog is open are silently
|
Subsequent unknown utterances while the dialog is open are silently
|
||||||
appended to the buffer so enrollment ultimately uses multiple clips.
|
appended to the buffer so enrollment ultimately uses multiple clips.
|
||||||
"""
|
"""
|
||||||
self._enroll_clip_buffer.append(audio)
|
self._enroll_clip_buffer.append(_PendingEnrollClip(
|
||||||
|
audio, int(sample_rate or self.config.audio.sample_rate), store_task,
|
||||||
|
))
|
||||||
if not self._enroll_ask_pending:
|
if not self._enroll_ask_pending:
|
||||||
self._enroll_ask_pending = True
|
self._enroll_ask_pending = True
|
||||||
# Use a stable key so the GUI can match confirm callbacks.
|
# Use a stable key so the GUI can match confirm callbacks.
|
||||||
|
|||||||
@@ -253,6 +253,11 @@ class SpeakerIDConfig:
|
|||||||
# auto-refinement. Only high-confidence identifications (score >=
|
# auto-refinement. Only high-confidence identifications (score >=
|
||||||
# threshold + refine_margin) update the profile, keeping noise clips out.
|
# threshold + refine_margin) update the profile, keeping noise clips out.
|
||||||
refine_margin: float = 0.10
|
refine_margin: float = 0.10
|
||||||
|
# Keep audio clips of unidentified speakers (WAV + embedding, stored under
|
||||||
|
# voice_profiles_dir/unidentified) so they can be played back and sorted
|
||||||
|
# into a profile from the GUI. Capped at max_unidentified (oldest dropped).
|
||||||
|
keep_unidentified: bool = True
|
||||||
|
max_unidentified: int = 50
|
||||||
# ── Unknown / guest speaker privileges ───────────────────────────────────
|
# ── Unknown / guest speaker privileges ───────────────────────────────────
|
||||||
# Applied when the speaker cannot be matched to any known voice profile.
|
# Applied when the speaker cannot be matched to any known voice profile.
|
||||||
# Controlled via the GUI "Settings → Speaker Identification" section.
|
# Controlled via the GUI "Settings → Speaker Identification" section.
|
||||||
@@ -295,7 +300,9 @@ class TrainingConfig:
|
|||||||
batch_size: int = 32
|
batch_size: int = 32
|
||||||
num_workers: int = 2
|
num_workers: int = 2
|
||||||
accelerator: str = "gpu" # gpu | cpu
|
accelerator: str = "gpu" # gpu | cpu
|
||||||
precision: str = "16-mixed" # 32 | 16-mixed | bf16-mixed
|
# bf16-mixed trains fastest and needs no loss scaling; GPUs without bf16
|
||||||
|
# fall back to 32 at launch.
|
||||||
|
precision: str = "bf16-mixed" # 32 | 16-mixed | bf16-mixed
|
||||||
language: str = "de" # espeak-ng voice code (de, en-us, ...)
|
language: str = "de" # espeak-ng voice code (de, en-us, ...)
|
||||||
|
|
||||||
|
|
||||||
@@ -430,6 +437,8 @@ def validate(cfg: "Config") -> None:
|
|||||||
cfg.speaker_id.threshold, 0.0, 1.0, 0.65, "speaker_id.threshold")
|
cfg.speaker_id.threshold, 0.0, 1.0, 0.65, "speaker_id.threshold")
|
||||||
cfg.speaker_id.refine_margin = _clamp(
|
cfg.speaker_id.refine_margin = _clamp(
|
||||||
cfg.speaker_id.refine_margin, 0.0, 0.5, 0.10, "speaker_id.refine_margin")
|
cfg.speaker_id.refine_margin, 0.0, 0.5, 0.10, "speaker_id.refine_margin")
|
||||||
|
cfg.speaker_id.max_unidentified = _clamp_int(
|
||||||
|
cfg.speaker_id.max_unidentified, 1, 500, 50, "speaker_id.max_unidentified")
|
||||||
cfg.wake_word.vad_gate = _clamp(
|
cfg.wake_word.vad_gate = _clamp(
|
||||||
cfg.wake_word.vad_gate, 0.0, 1.0, 0.0, "wake_word.vad_gate")
|
cfg.wake_word.vad_gate, 0.0, 1.0, 0.0, "wake_word.vad_gate")
|
||||||
cfg.wake_word.voice_gate_ratio = _clamp(
|
cfg.wake_word.voice_gate_ratio = _clamp(
|
||||||
@@ -512,8 +521,8 @@ def validate(cfg: "Config") -> None:
|
|||||||
if t.precision not in {"32", "16-mixed", "bf16-mixed"}:
|
if t.precision not in {"32", "16-mixed", "bf16-mixed"}:
|
||||||
log.warning(
|
log.warning(
|
||||||
f"config: training.precision='{t.precision}' is unknown "
|
f"config: training.precision='{t.precision}' is unknown "
|
||||||
"(32|16-mixed|bf16-mixed), using '16-mixed'")
|
"(32|16-mixed|bf16-mixed), using 'bf16-mixed'")
|
||||||
t.precision = "16-mixed"
|
t.precision = "bf16-mixed"
|
||||||
|
|
||||||
if cfg.memory.provider not in {"ollama", "hermes"}:
|
if cfg.memory.provider not in {"ollama", "hermes"}:
|
||||||
log.warning(
|
log.warning(
|
||||||
|
|||||||
+855
-30
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,186 @@
|
|||||||
|
"""Inbox of voice clips from speakers that speaker ID could not match.
|
||||||
|
|
||||||
|
Each clip is kept so the user can listen to it in the GUI and sort it into an
|
||||||
|
existing voice profile or use it to create a new one::
|
||||||
|
|
||||||
|
~/.javis/voice_profiles/unidentified/{id}.wav 16-bit PCM audio
|
||||||
|
~/.javis/voice_profiles/unidentified/{id}.npy ECAPA embedding (optional)
|
||||||
|
~/.javis/voice_profiles/unidentified/index.json metadata, oldest first
|
||||||
|
|
||||||
|
The inbox is capped at ``max_clips``; the oldest clips are dropped first.
|
||||||
|
All methods are thread-safe (clips are added from worker threads).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ..core.logger import get_logger
|
||||||
|
|
||||||
|
log = get_logger("unidentified")
|
||||||
|
|
||||||
|
_INDEX_FILE = "index.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _mono(audio: np.ndarray) -> np.ndarray:
|
||||||
|
a = np.asarray(audio, dtype=np.float32)
|
||||||
|
if a.ndim > 1:
|
||||||
|
a = a.mean(axis=1)
|
||||||
|
return a.reshape(-1)
|
||||||
|
|
||||||
|
|
||||||
|
class UnidentifiedStore:
|
||||||
|
"""Bounded on-disk store of unidentified speaker clips."""
|
||||||
|
|
||||||
|
def __init__(self, directory: Path, max_clips: int = 50) -> None:
|
||||||
|
self.dir = Path(directory)
|
||||||
|
self.max_clips = max(1, int(max_clips))
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._index: list[dict] | None = None # lazily loaded
|
||||||
|
|
||||||
|
# ── index I/O ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _entries(self) -> list[dict]:
|
||||||
|
if self._index is None:
|
||||||
|
idx = self.dir / _INDEX_FILE
|
||||||
|
data: list = []
|
||||||
|
if idx.exists():
|
||||||
|
try:
|
||||||
|
data = json.loads(idx.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(f"[unidentified] index read failed: {exc}")
|
||||||
|
self._index = [
|
||||||
|
e for e in data
|
||||||
|
if isinstance(e, dict) and e.get("id") and e.get("file")
|
||||||
|
and (self.dir / e["file"]).exists()
|
||||||
|
]
|
||||||
|
return self._index
|
||||||
|
|
||||||
|
def _save(self) -> None:
|
||||||
|
self.dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
idx = self.dir / _INDEX_FILE
|
||||||
|
tmp = idx.with_suffix(".json.part")
|
||||||
|
try:
|
||||||
|
tmp.write_text(
|
||||||
|
json.dumps(self._entries(), ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
tmp.replace(idx)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.error(f"[unidentified] index write failed: {exc}")
|
||||||
|
|
||||||
|
def _delete_files(self, entry: dict) -> None:
|
||||||
|
for key in ("file", "embedding"):
|
||||||
|
fname = entry.get(key)
|
||||||
|
if not fname:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
(self.dir / fname).unlink(missing_ok=True)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(f"[unidentified] could not delete {fname}: {exc}")
|
||||||
|
|
||||||
|
# ── public API ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def add(
|
||||||
|
self,
|
||||||
|
audio: np.ndarray,
|
||||||
|
sample_rate: int,
|
||||||
|
*,
|
||||||
|
embedding: np.ndarray | None = None,
|
||||||
|
text: str = "",
|
||||||
|
source: str = "",
|
||||||
|
best_name: str | None = None,
|
||||||
|
best_score: float | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Store a clip and return its id. Prunes the oldest beyond the cap."""
|
||||||
|
import soundfile as sf
|
||||||
|
|
||||||
|
mono = np.clip(_mono(audio), -1.0, 1.0)
|
||||||
|
clip_id = f"{time.strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:6]}"
|
||||||
|
with self._lock:
|
||||||
|
self.dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
wav_name = f"{clip_id}.wav"
|
||||||
|
sf.write(str(self.dir / wav_name), mono, int(sample_rate), subtype="PCM_16")
|
||||||
|
emb_name = None
|
||||||
|
if embedding is not None and np.any(embedding):
|
||||||
|
emb_name = f"{clip_id}.npy"
|
||||||
|
np.save(str(self.dir / emb_name),
|
||||||
|
np.asarray(embedding, dtype=np.float32).reshape(-1))
|
||||||
|
entries = self._entries()
|
||||||
|
entries.append({
|
||||||
|
"id": clip_id,
|
||||||
|
"file": wav_name,
|
||||||
|
"embedding": emb_name,
|
||||||
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
|
"duration_s": round(len(mono) / float(sample_rate), 2),
|
||||||
|
"sample_rate": int(sample_rate),
|
||||||
|
"text": (text or "").strip(),
|
||||||
|
"source": source or "",
|
||||||
|
"best_name": best_name,
|
||||||
|
"best_score": None if best_score is None else round(float(best_score), 4),
|
||||||
|
})
|
||||||
|
while len(entries) > self.max_clips:
|
||||||
|
self._delete_files(entries.pop(0))
|
||||||
|
self._save()
|
||||||
|
log.info(f"[unidentified] stored clip {clip_id} ({len(mono) / sample_rate:.1f}s)")
|
||||||
|
return clip_id
|
||||||
|
|
||||||
|
def entries(self) -> list[dict]:
|
||||||
|
"""All clip metadata, newest first (copies — safe to keep)."""
|
||||||
|
with self._lock:
|
||||||
|
return [dict(e) for e in reversed(self._entries())]
|
||||||
|
|
||||||
|
def get(self, clip_id: str) -> dict | None:
|
||||||
|
with self._lock:
|
||||||
|
entry = next((e for e in self._entries() if e["id"] == clip_id), None)
|
||||||
|
return dict(entry) if entry else None
|
||||||
|
|
||||||
|
def load_audio(self, clip_id: str) -> tuple[np.ndarray, int]:
|
||||||
|
"""Return ``(float32 mono audio, sample_rate)``; raises ``KeyError`` if unknown."""
|
||||||
|
import soundfile as sf
|
||||||
|
|
||||||
|
entry = self.get(clip_id)
|
||||||
|
if entry is None:
|
||||||
|
raise KeyError(clip_id)
|
||||||
|
audio, sr = sf.read(str(self.dir / entry["file"]), dtype="float32", always_2d=False)
|
||||||
|
return _mono(audio), int(sr)
|
||||||
|
|
||||||
|
def load_embedding(self, clip_id: str) -> np.ndarray | None:
|
||||||
|
entry = self.get(clip_id)
|
||||||
|
if entry is None or not entry.get("embedding"):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return np.load(str(self.dir / entry["embedding"])).astype(np.float32)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(f"[unidentified] embedding for {clip_id} unreadable: {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def remove(self, clip_ids: Iterable[str]) -> int:
|
||||||
|
"""Delete the given clips; returns how many were removed."""
|
||||||
|
wanted = set(clip_ids)
|
||||||
|
if not wanted:
|
||||||
|
return 0
|
||||||
|
with self._lock:
|
||||||
|
entries = self._entries()
|
||||||
|
gone = [e for e in entries if e["id"] in wanted]
|
||||||
|
if not gone:
|
||||||
|
return 0
|
||||||
|
for e in gone:
|
||||||
|
self._delete_files(e)
|
||||||
|
self._index = [e for e in entries if e["id"] not in wanted]
|
||||||
|
self._save()
|
||||||
|
return len(gone)
|
||||||
|
|
||||||
|
def clear(self) -> int:
|
||||||
|
with self._lock:
|
||||||
|
return self.remove([e["id"] for e in self._entries()])
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
with self._lock:
|
||||||
|
return len(self._entries())
|
||||||
+180
-88
@@ -47,6 +47,16 @@ def _l2(v: np.ndarray) -> np.ndarray:
|
|||||||
return v / n if n > 0 else v
|
return v / n if n > 0 else v
|
||||||
|
|
||||||
|
|
||||||
|
def _as_vector(embedding) -> Optional[np.ndarray]:
|
||||||
|
"""Flatten *embedding* to float32; ``None`` for missing or all-zero vectors."""
|
||||||
|
if embedding is None:
|
||||||
|
return None
|
||||||
|
vec = np.asarray(embedding, dtype=np.float32).reshape(-1)
|
||||||
|
if vec.size == 0 or not np.any(vec):
|
||||||
|
return None
|
||||||
|
return vec
|
||||||
|
|
||||||
|
|
||||||
# Minimum utterance duration for a reliable embedding. Shorter clips are
|
# Minimum utterance duration for a reliable embedding. Shorter clips are
|
||||||
# passed through but logged as a warning — ECAPA was trained on ≥ 1 s segments.
|
# passed through but logged as a warning — ECAPA was trained on ≥ 1 s segments.
|
||||||
_MIN_EMBED_DURATION_S = 0.5
|
_MIN_EMBED_DURATION_S = 0.5
|
||||||
@@ -125,6 +135,28 @@ class NullProfiler:
|
|||||||
def identify(self, audio_np: np.ndarray, sr: int) -> Optional[tuple[str, float]]: # noqa: ARG002
|
def identify(self, audio_np: np.ndarray, sr: int) -> Optional[tuple[str, float]]: # noqa: ARG002
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def identify_embedding(self, embedding) -> Optional[tuple[str, float]]: # noqa: ARG002
|
||||||
|
return None
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def has_profile(self, name: str) -> bool: # noqa: ARG002
|
||||||
|
return False
|
||||||
|
|
||||||
|
def rank(self, embedding) -> list[tuple[str, float]]: # noqa: ARG002
|
||||||
|
return []
|
||||||
|
|
||||||
|
def enroll_embeddings(self, name: str, embeddings: list) -> bool: # noqa: ARG002
|
||||||
|
log.warning("[profiler] NullProfiler — speechbrain unavailable, enroll skipped")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def refine(self, name: str, clip: np.ndarray, sr: int) -> None: # noqa: ARG002
|
||||||
|
pass
|
||||||
|
|
||||||
|
def refine_embedding(self, name: str, embedding) -> bool: # noqa: ARG002
|
||||||
|
return False
|
||||||
|
|
||||||
def list_profiles(self) -> list[dict]:
|
def list_profiles(self) -> list[dict]:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@@ -155,8 +187,13 @@ class VoiceProfiler:
|
|||||||
self.config = config
|
self.config = config
|
||||||
self._profiles_dir: Path = config.paths.voice_profiles_dir
|
self._profiles_dir: Path = config.paths.voice_profiles_dir
|
||||||
self._encoder = None # speechbrain EncoderClassifier or None
|
self._encoder = None # speechbrain EncoderClassifier or None
|
||||||
|
self._device = "cpu" # device the encoder runs on
|
||||||
|
self._encoder_lock = threading.Lock() # serialises the CPU fallback
|
||||||
self._profiles: dict[str, np.ndarray] = {} # name → L2-normed embedding
|
self._profiles: dict[str, np.ndarray] = {} # name → L2-normed embedding
|
||||||
self._index: list[dict] = [] # metadata records
|
self._index: list[dict] = [] # metadata records
|
||||||
|
# identify/refine run on worker threads while the GUI enrolls/deletes
|
||||||
|
# on the loop thread — serialise access to _profiles/_index.
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
# ── lifecycle ─────────────────────────────────────────────────────────────
|
# ── lifecycle ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -164,9 +201,18 @@ class VoiceProfiler:
|
|||||||
"""Load the ECAPA-TDNN encoder and all enrolled profiles from disk."""
|
"""Load the ECAPA-TDNN encoder and all enrolled profiles from disk."""
|
||||||
self._profiles_dir.mkdir(parents=True, exist_ok=True)
|
self._profiles_dir.mkdir(parents=True, exist_ok=True)
|
||||||
self._load_encoder()
|
self._load_encoder()
|
||||||
self._load_index()
|
with self._lock:
|
||||||
|
self._load_index()
|
||||||
log.info(f"[profiler] ready — {len(self._profiles)} profile(s) loaded")
|
log.info(f"[profiler] ready — {len(self._profiles)} profile(s) loaded")
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
"""``True`` once the ECAPA encoder is loaded (embeddings are meaningful)."""
|
||||||
|
return self._encoder is not None
|
||||||
|
|
||||||
|
def has_profile(self, name: str) -> bool:
|
||||||
|
with self._lock:
|
||||||
|
return name in self._profiles
|
||||||
|
|
||||||
def _load_encoder(self) -> None:
|
def _load_encoder(self) -> None:
|
||||||
try:
|
try:
|
||||||
import os
|
import os
|
||||||
@@ -187,8 +233,6 @@ class VoiceProfiler:
|
|||||||
category=UserWarning,
|
category=UserWarning,
|
||||||
)
|
)
|
||||||
# 3. speechbrain Pretrainer symlinks — harmless on Windows because
|
# 3. speechbrain Pretrainer symlinks — harmless on Windows because
|
||||||
self._device = "cpu" # device the encoder runs on
|
|
||||||
self._encoder_lock = threading.Lock() # serialises the CPU fallback
|
|
||||||
# model files are copied rather than symlinked.
|
# model files are copied rather than symlinked.
|
||||||
warnings.filterwarnings(
|
warnings.filterwarnings(
|
||||||
"ignore",
|
"ignore",
|
||||||
@@ -210,6 +254,33 @@ class VoiceProfiler:
|
|||||||
)
|
)
|
||||||
self._encoder = None
|
self._encoder = None
|
||||||
|
|
||||||
|
def _build_encoder(self, device: str):
|
||||||
|
from speechbrain.inference.classifiers import EncoderClassifier
|
||||||
|
|
||||||
|
savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
|
||||||
|
log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
|
||||||
|
return EncoderClassifier.from_hparams(
|
||||||
|
source=_MODEL_SOURCE,
|
||||||
|
savedir=savedir,
|
||||||
|
run_opts={"device": device},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fallback_to_cpu(self, failed, exc: BaseException):
|
||||||
|
"""Swap a broken CUDA encoder for a CPU one (once, thread-safe).
|
||||||
|
|
||||||
|
CUDA errors here usually mean the context died (e.g. the NVIDIA driver
|
||||||
|
was updated while Javis was running), which never recovers in-process.
|
||||||
|
"""
|
||||||
|
with self._encoder_lock:
|
||||||
|
if self._encoder is failed:
|
||||||
|
log.warning(
|
||||||
|
f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; "
|
||||||
|
"restart Javis to use the GPU again"
|
||||||
|
)
|
||||||
|
self._encoder = self._build_encoder("cpu")
|
||||||
|
self._device = "cpu"
|
||||||
|
return self._encoder
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _try_cuda() -> bool:
|
def _try_cuda() -> bool:
|
||||||
try:
|
try:
|
||||||
@@ -254,33 +325,6 @@ class VoiceProfiler:
|
|||||||
|
|
||||||
# ── embedding ─────────────────────────────────────────────────────────────
|
# ── embedding ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _build_encoder(self, device: str):
|
|
||||||
from speechbrain.inference.classifiers import EncoderClassifier
|
|
||||||
|
|
||||||
savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
|
|
||||||
log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
|
|
||||||
return EncoderClassifier.from_hparams(
|
|
||||||
source=_MODEL_SOURCE,
|
|
||||||
savedir=savedir,
|
|
||||||
run_opts={"device": device},
|
|
||||||
)
|
|
||||||
|
|
||||||
def _fallback_to_cpu(self, failed, exc: BaseException):
|
|
||||||
"""Swap a broken CUDA encoder for a CPU one (once, thread-safe).
|
|
||||||
|
|
||||||
CUDA errors here usually mean the context died (e.g. the NVIDIA driver
|
|
||||||
was updated while Javis was running), which never recovers in-process.
|
|
||||||
"""
|
|
||||||
with self._encoder_lock:
|
|
||||||
if self._encoder is failed:
|
|
||||||
log.warning(
|
|
||||||
f"[profiler] CUDA failed ({exc}) — switching speaker ID to CPU; "
|
|
||||||
"restart Javis to use the GPU again"
|
|
||||||
)
|
|
||||||
self._encoder = self._build_encoder("cpu")
|
|
||||||
self._device = "cpu"
|
|
||||||
return self._encoder
|
|
||||||
|
|
||||||
def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
|
def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
|
||||||
"""Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*."""
|
"""Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*."""
|
||||||
if self._encoder is None:
|
if self._encoder is None:
|
||||||
@@ -328,25 +372,58 @@ class VoiceProfiler:
|
|||||||
if self._encoder is None:
|
if self._encoder is None:
|
||||||
log.warning("[profiler] enroll skipped — encoder not loaded")
|
log.warning("[profiler] enroll skipped — encoder not loaded")
|
||||||
return
|
return
|
||||||
vecs = [self.embed(c, sr) for c in clips]
|
self.enroll_embeddings(name, [self.embed(c, sr) for c in clips])
|
||||||
avg = _l2(np.mean(vecs, axis=0))
|
|
||||||
|
|
||||||
# Build a filesystem-safe filename from the profile name.
|
def enroll_embeddings(self, name: str, embeddings: list[np.ndarray]) -> bool:
|
||||||
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in name)
|
"""Create (or replace) *name*'s profile from precomputed embeddings.
|
||||||
npy_path = self._profiles_dir / f"{safe}.npy"
|
|
||||||
np.save(str(npy_path), avg)
|
|
||||||
self._profiles[name] = avg
|
|
||||||
|
|
||||||
# Replace existing entry if the name already exists.
|
All-zero vectors (from an unloaded encoder) are ignored. Returns
|
||||||
self._index = [e for e in self._index if e["name"] != name]
|
``False`` when no usable embedding remains.
|
||||||
self._index.append({
|
"""
|
||||||
"name": name,
|
vecs = [v for v in (_as_vector(e) for e in embeddings) if v is not None]
|
||||||
"file": f"{safe}.npy",
|
if not vecs:
|
||||||
"enrolled_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
log.warning(f"[profiler] enroll '{name}' skipped — no usable embeddings")
|
||||||
"clip_count": len(clips),
|
return False
|
||||||
})
|
avg = _l2(np.mean(vecs, axis=0)).astype(np.float32)
|
||||||
self._save_index()
|
|
||||||
log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)")
|
with self._lock:
|
||||||
|
self._profiles_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
old = next((e for e in self._index if e["name"] == name), None)
|
||||||
|
if old is not None:
|
||||||
|
fname = old["file"]
|
||||||
|
else:
|
||||||
|
# Filesystem-safe filename; never reuse another profile's file
|
||||||
|
# ("Anna B" and "Anna_B" sanitise to the same stem).
|
||||||
|
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in name)
|
||||||
|
used = {e["file"] for e in self._index}
|
||||||
|
fname, i = f"{safe}.npy", 2
|
||||||
|
while fname in used:
|
||||||
|
fname, i = f"{safe}_{i}.npy", i + 1
|
||||||
|
np.save(str(self._profiles_dir / fname), avg)
|
||||||
|
self._profiles[name] = avg
|
||||||
|
|
||||||
|
# Replace existing entry if the name already exists.
|
||||||
|
self._index = [e for e in self._index if e["name"] != name]
|
||||||
|
self._index.append({
|
||||||
|
"name": name,
|
||||||
|
"file": fname,
|
||||||
|
"enrolled_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
|
"clip_count": len(vecs),
|
||||||
|
})
|
||||||
|
self._save_index()
|
||||||
|
log.info(f"[profiler] enrolled '{name}' from {len(vecs)} clip(s)")
|
||||||
|
return True
|
||||||
|
|
||||||
|
def rank(self, embedding) -> list[tuple[str, float]]:
|
||||||
|
"""Cosine similarity of *embedding* to every profile, best first."""
|
||||||
|
vec = _as_vector(embedding)
|
||||||
|
if vec is None:
|
||||||
|
return []
|
||||||
|
with self._lock:
|
||||||
|
items = list(self._profiles.items())
|
||||||
|
scores = [(n, _cosine(vec, e)) for n, e in items if e.shape == vec.shape]
|
||||||
|
scores.sort(key=lambda x: -x[1])
|
||||||
|
return scores
|
||||||
|
|
||||||
def identify(
|
def identify(
|
||||||
self, audio_np: np.ndarray, sr: int
|
self, audio_np: np.ndarray, sr: int
|
||||||
@@ -358,20 +435,19 @@ class VoiceProfiler:
|
|||||||
"""
|
"""
|
||||||
if self._encoder is None or not self._profiles:
|
if self._encoder is None or not self._profiles:
|
||||||
return None
|
return None
|
||||||
query = self.embed(audio_np, sr)
|
return self.identify_embedding(self.embed(audio_np, sr))
|
||||||
threshold = self.config.speaker_id.threshold
|
|
||||||
|
|
||||||
scores: dict[str, float] = {
|
def identify_embedding(self, embedding) -> Optional[tuple[str, float]]:
|
||||||
name: _cosine(query, emb) for name, emb in self._profiles.items()
|
"""Like :meth:`identify` for an embedding computed by :meth:`embed`."""
|
||||||
}
|
ranked = self.rank(embedding)
|
||||||
best_name = max(scores, key=scores.__getitem__)
|
if not ranked:
|
||||||
best_score = scores[best_name]
|
return None
|
||||||
|
threshold = self.config.speaker_id.threshold
|
||||||
|
best_name, best_score = ranked[0]
|
||||||
|
|
||||||
# Always log all candidate scores so identification failures are visible
|
# Always log all candidate scores so identification failures are visible
|
||||||
# in normal (INFO-level) logs without needing to enable DEBUG mode.
|
# in normal (INFO-level) logs without needing to enable DEBUG mode.
|
||||||
score_str = ", ".join(
|
score_str = ", ".join(f"{n}={s:.3f}" for n, s in ranked)
|
||||||
f"{n}={s:.3f}" for n, s in sorted(scores.items(), key=lambda x: -x[1])
|
|
||||||
)
|
|
||||||
if best_score >= threshold:
|
if best_score >= threshold:
|
||||||
log.info(f"[profiler] identified '{best_name}' [{score_str}] threshold={threshold}")
|
log.info(f"[profiler] identified '{best_name}' [{score_str}] threshold={threshold}")
|
||||||
return best_name, float(best_score)
|
return best_name, float(best_score)
|
||||||
@@ -382,7 +458,8 @@ class VoiceProfiler:
|
|||||||
|
|
||||||
def list_profiles(self) -> list[dict]:
|
def list_profiles(self) -> list[dict]:
|
||||||
"""Return a list of profile metadata dicts from the index."""
|
"""Return a list of profile metadata dicts from the index."""
|
||||||
return list(self._index)
|
with self._lock:
|
||||||
|
return list(self._index)
|
||||||
|
|
||||||
def refine(self, name: str, clip: np.ndarray, sr: int) -> None:
|
def refine(self, name: str, clip: np.ndarray, sr: int) -> None:
|
||||||
"""Blend a new voice sample into *name*'s existing profile.
|
"""Blend a new voice sample into *name*'s existing profile.
|
||||||
@@ -393,36 +470,50 @@ class VoiceProfiler:
|
|||||||
"""
|
"""
|
||||||
if self._encoder is None:
|
if self._encoder is None:
|
||||||
return
|
return
|
||||||
entry = next((e for e in self._index if e["name"] == name), None)
|
if not self.has_profile(name):
|
||||||
if entry is None or name not in self._profiles:
|
|
||||||
log.warning(f"[profiler] refine: no profile found for '{name}'")
|
log.warning(f"[profiler] refine: no profile found for '{name}'")
|
||||||
return
|
return
|
||||||
new_emb = self.embed(clip, sr)
|
self.refine_embedding(name, self.embed(clip, sr))
|
||||||
n = entry.get("clip_count", 1)
|
|
||||||
current = self._profiles[name]
|
def refine_embedding(self, name: str, embedding) -> bool:
|
||||||
# Running centroid: (current * n + new) / (n+1), then L2-normalise.
|
"""Blend a precomputed embedding into *name*'s centroid (see :meth:`refine`)."""
|
||||||
blended = _l2(current * n + new_emb)
|
new_emb = _as_vector(embedding)
|
||||||
npy_path = self._profiles_dir / entry["file"]
|
if new_emb is None:
|
||||||
np.save(str(npy_path), blended)
|
return False
|
||||||
self._profiles[name] = blended
|
with self._lock:
|
||||||
entry["clip_count"] = n + 1
|
entry = next((e for e in self._index if e["name"] == name), None)
|
||||||
entry["last_seen"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
if entry is None or name not in self._profiles:
|
||||||
self._save_index()
|
log.warning(f"[profiler] refine: no profile found for '{name}'")
|
||||||
|
return False
|
||||||
|
n = entry.get("clip_count", 1)
|
||||||
|
current = self._profiles[name]
|
||||||
|
if current.shape != new_emb.shape:
|
||||||
|
log.warning(f"[profiler] refine '{name}': embedding shape mismatch")
|
||||||
|
return False
|
||||||
|
# Running centroid: (current * n + new) / (n+1), then L2-normalise.
|
||||||
|
blended = _l2(current * n + new_emb).astype(np.float32)
|
||||||
|
np.save(str(self._profiles_dir / entry["file"]), blended)
|
||||||
|
self._profiles[name] = blended
|
||||||
|
entry["clip_count"] = n + 1
|
||||||
|
entry["last_seen"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
|
self._save_index()
|
||||||
log.debug(f"[profiler] refined '{name}' — centroid now over {n + 1} clip(s)")
|
log.debug(f"[profiler] refined '{name}' — centroid now over {n + 1} clip(s)")
|
||||||
|
return True
|
||||||
|
|
||||||
def delete_profile(self, name: str) -> bool:
|
def delete_profile(self, name: str) -> bool:
|
||||||
"""Remove *name*'s embedding from disk and memory."""
|
"""Remove *name*'s embedding from disk and memory."""
|
||||||
entry = next((e for e in self._index if e["name"] == name), None)
|
with self._lock:
|
||||||
if entry is None:
|
entry = next((e for e in self._index if e["name"] == name), None)
|
||||||
return False
|
if entry is None:
|
||||||
npy = self._profiles_dir / entry["file"]
|
return False
|
||||||
try:
|
npy = self._profiles_dir / entry["file"]
|
||||||
npy.unlink(missing_ok=True)
|
try:
|
||||||
except Exception as exc:
|
npy.unlink(missing_ok=True)
|
||||||
log.warning(f"[profiler] delete {npy.name} failed: {exc}")
|
except Exception as exc:
|
||||||
self._index = [e for e in self._index if e["name"] != name]
|
log.warning(f"[profiler] delete {npy.name} failed: {exc}")
|
||||||
self._profiles.pop(name, None)
|
self._index = [e for e in self._index if e["name"] != name]
|
||||||
self._save_index()
|
self._profiles.pop(name, None)
|
||||||
|
self._save_index()
|
||||||
log.info(f"[profiler] deleted profile '{name}'")
|
log.info(f"[profiler] deleted profile '{name}'")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -437,10 +528,11 @@ class VoiceProfiler:
|
|||||||
|
|
||||||
def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None:
|
def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None:
|
||||||
"""Persist updated *privileges* for *name* to ``index.json``."""
|
"""Persist updated *privileges* for *name* to ``index.json``."""
|
||||||
entry = next((e for e in self._index if e["name"] == name), None)
|
with self._lock:
|
||||||
if entry is None:
|
entry = next((e for e in self._index if e["name"] == name), None)
|
||||||
log.warning(f"[profiler] set_privileges: no profile found for '{name}'")
|
if entry is None:
|
||||||
return
|
log.warning(f"[profiler] set_privileges: no profile found for '{name}'")
|
||||||
entry["privileges"] = privileges.to_dict()
|
return
|
||||||
self._save_index()
|
entry["privileges"] = privileges.to_dict()
|
||||||
|
self._save_index()
|
||||||
log.info(f"[profiler] privileges updated for '{name}'")
|
log.info(f"[profiler] privileges updated for '{name}'")
|
||||||
+129
-34
@@ -12,6 +12,11 @@ The model path is resolved in this order:
|
|||||||
1. ``config.tts.piper_model`` (explicit path in config.yaml)
|
1. ``config.tts.piper_model`` (explicit path in config.yaml)
|
||||||
2. ``{voices_dir}/{tts.default_voice}/piper_model.onnx`` (active voice dir)
|
2. ``{voices_dir}/{tts.default_voice}/piper_model.onnx`` (active voice dir)
|
||||||
|
|
||||||
|
The model is reloaded by itself when that changes: when training updates the
|
||||||
|
voice's model (it does every few minutes, see ``javis.voices.piper_train``) or
|
||||||
|
another voice is activated. Until the new model is loaded, the previous one
|
||||||
|
keeps speaking.
|
||||||
|
|
||||||
Inference is synchronous (ONNX runtime) so it runs in a thread-pool executor
|
Inference is synchronous (ONNX runtime) so it runs in a thread-pool executor
|
||||||
to avoid blocking the asyncio event loop.
|
to avoid blocking the asyncio event loop.
|
||||||
|
|
||||||
@@ -23,6 +28,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import re
|
import re
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -59,42 +65,42 @@ def _clean(text: str) -> str:
|
|||||||
class PiperBackend(TTSBackend):
|
class PiperBackend(TTSBackend):
|
||||||
"""Piper ONNX TTS backend."""
|
"""Piper ONNX TTS backend."""
|
||||||
|
|
||||||
|
# Synthesis looks for a changed model at most this often.
|
||||||
|
CHECK_SECONDS = 2.0
|
||||||
|
|
||||||
def __init__(self, config: Config) -> None:
|
def __init__(self, config: Config) -> None:
|
||||||
self.config = config
|
self.config = config
|
||||||
self._voice: Optional[object] = None # piper.voice.PiperVoice
|
self._voice: Optional[object] = None # piper.voice.PiperVoice
|
||||||
self._sample_rate: int = 22050
|
self._sample_rate: int = 22050
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock() # one synthesis at a time; guards the swap
|
||||||
|
self._reloading = threading.Lock()
|
||||||
|
self._loaded: tuple | None = None # _files_signature() of the loaded model
|
||||||
|
self._failed: tuple | None = None # ... of a model that failed to load
|
||||||
|
self._checked = 0.0
|
||||||
|
|
||||||
# ── TTSBackend interface ──────────────────────────────────────────────────
|
# ── TTSBackend interface ──────────────────────────────────────────────────
|
||||||
|
|
||||||
def load(self) -> None:
|
def load(self) -> None:
|
||||||
"""Load the ONNX model. Called once at startup."""
|
"""Load the ONNX model. Called once at startup."""
|
||||||
try:
|
|
||||||
from piper.voice import PiperVoice # type: ignore[import]
|
|
||||||
except ImportError as exc:
|
|
||||||
raise RuntimeError(
|
|
||||||
"piper-tts is not installed. Run: pip install piper-tts"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
model_path = self._resolve_model_path()
|
model_path = self._resolve_model_path()
|
||||||
config_path = Path(str(model_path) + ".json")
|
signature = _files_signature(model_path)
|
||||||
|
try:
|
||||||
log.info(f"Loading Piper model: {model_path}")
|
voice = self._load_voice(model_path)
|
||||||
self._voice = PiperVoice.load(
|
except BaseException:
|
||||||
str(model_path),
|
self._failed = signature
|
||||||
config_path=str(config_path) if config_path.exists() else None,
|
raise
|
||||||
use_cuda=False, # ONNX runtime picks CUDA automatically when available
|
with self._lock:
|
||||||
)
|
self._install(voice, signature)
|
||||||
self._sample_rate = (
|
|
||||||
self._voice.config.sample_rate # type: ignore[union-attr]
|
|
||||||
if hasattr(self._voice, "config")
|
|
||||||
else 22050
|
|
||||||
)
|
|
||||||
log.info(
|
log.info(
|
||||||
f"Piper model loaded — sample_rate={self._sample_rate} Hz, "
|
f"Piper model loaded — sample_rate={self._sample_rate} Hz, "
|
||||||
f"language={getattr(getattr(self._voice, 'config', None), 'espeak_voice', '?')}"
|
f"language={getattr(getattr(voice, 'config', None), 'espeak_voice', '?')}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def model_available(self) -> bool:
|
||||||
|
"""Whether there is a model for :meth:`load` (other than one it failed to load)."""
|
||||||
|
model_path = self._model_path()
|
||||||
|
return model_path is not None and _files_signature(model_path) != self._failed
|
||||||
|
|
||||||
async def synthesize(
|
async def synthesize(
|
||||||
self,
|
self,
|
||||||
text: str,
|
text: str,
|
||||||
@@ -108,13 +114,17 @@ class PiperBackend(TTSBackend):
|
|||||||
sample_rate=self._sample_rate,
|
sample_rate=self._sample_rate,
|
||||||
)
|
)
|
||||||
|
|
||||||
audio = await asyncio.to_thread(self._synthesize_sync, text)
|
audio, sample_rate = await asyncio.to_thread(self._synthesize_sync, text)
|
||||||
return SynthesisResult(audio=audio, sample_rate=self._sample_rate)
|
return SynthesisResult(audio=audio, sample_rate=sample_rate)
|
||||||
|
|
||||||
# ── Internal ──────────────────────────────────────────────────────────────
|
# ── Internal ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _synthesize_sync(self, text: str) -> np.ndarray:
|
def _synthesize_sync(self, text: str) -> tuple[np.ndarray, int]:
|
||||||
"""Run Piper inference synchronously (called from thread-pool)."""
|
"""Run Piper inference synchronously (called from thread-pool).
|
||||||
|
|
||||||
|
Returns the audio and its sample rate.
|
||||||
|
"""
|
||||||
|
self._check_for_new_model()
|
||||||
if self._voice is None:
|
if self._voice is None:
|
||||||
raise RuntimeError("PiperBackend.load() has not been called.")
|
raise RuntimeError("PiperBackend.load() has not been called.")
|
||||||
|
|
||||||
@@ -128,31 +138,103 @@ class PiperBackend(TTSBackend):
|
|||||||
|
|
||||||
chunks: list[np.ndarray] = []
|
chunks: list[np.ndarray] = []
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
sample_rate = self._sample_rate
|
||||||
for chunk in self._voice.synthesize(text, syn_config):
|
for chunk in self._voice.synthesize(text, syn_config):
|
||||||
arr = chunk.audio_float_array
|
arr = chunk.audio_float_array
|
||||||
if arr is not None and len(arr) > 0:
|
if arr is not None and len(arr) > 0:
|
||||||
chunks.append(np.asarray(arr, dtype=np.float32))
|
chunks.append(np.asarray(arr, dtype=np.float32))
|
||||||
|
|
||||||
if not chunks:
|
if not chunks:
|
||||||
return np.zeros(0, dtype=np.float32)
|
return np.zeros(0, dtype=np.float32), sample_rate
|
||||||
return np.concatenate(chunks)
|
return np.concatenate(chunks), sample_rate
|
||||||
|
|
||||||
def _resolve_model_path(self) -> Path:
|
@staticmethod
|
||||||
"""Return the ONNX model path from config or voice profile dir."""
|
def _load_voice(model_path: Path):
|
||||||
|
try:
|
||||||
|
from piper.voice import PiperVoice # type: ignore[import]
|
||||||
|
except ImportError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"piper-tts is not installed. Run: pip install piper-tts"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
config_path = Path(str(model_path) + ".json")
|
||||||
|
log.info(f"Loading Piper model: {model_path}")
|
||||||
|
return PiperVoice.load(
|
||||||
|
str(model_path),
|
||||||
|
config_path=str(config_path) if config_path.exists() else None,
|
||||||
|
use_cuda=False, # ONNX runtime picks CUDA automatically when available
|
||||||
|
)
|
||||||
|
|
||||||
|
def _install(self, voice, signature: tuple | None) -> None:
|
||||||
|
"""Make *voice* the one that speaks (call with ``_lock`` held)."""
|
||||||
|
self._voice = voice
|
||||||
|
self._sample_rate = (
|
||||||
|
voice.config.sample_rate # type: ignore[union-attr]
|
||||||
|
if hasattr(voice, "config")
|
||||||
|
else 22050
|
||||||
|
)
|
||||||
|
self._loaded, self._failed = signature, None
|
||||||
|
|
||||||
|
def _check_for_new_model(self) -> None:
|
||||||
|
"""Start loading the model in the background if it changed since it was loaded."""
|
||||||
|
if self._voice is None or time.monotonic() - self._checked < self.CHECK_SECONDS:
|
||||||
|
return
|
||||||
|
self._checked = time.monotonic()
|
||||||
|
model_path = self._model_path()
|
||||||
|
if model_path is None:
|
||||||
|
return
|
||||||
|
signature = _files_signature(model_path)
|
||||||
|
if signature is None or signature in (self._loaded, self._failed):
|
||||||
|
return
|
||||||
|
if not self._reloading.acquire(blocking=False):
|
||||||
|
return # being loaded already
|
||||||
|
try:
|
||||||
|
threading.Thread(target=self._reload, args=(model_path, signature),
|
||||||
|
name="piper-reload", daemon=True).start()
|
||||||
|
except BaseException:
|
||||||
|
self._reloading.release()
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _reload(self, model_path: Path, signature: tuple) -> None:
|
||||||
|
try:
|
||||||
|
voice = self._load_voice(model_path)
|
||||||
|
for _ in voice.synthesize("Bereit."): # warm up before it speaks
|
||||||
|
pass
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep speaking with the loaded model
|
||||||
|
self._failed = signature
|
||||||
|
log.warning(f"Could not load the changed Piper model {model_path} ({exc}); "
|
||||||
|
"keeping the current one.")
|
||||||
|
else:
|
||||||
|
with self._lock:
|
||||||
|
self._install(voice, signature)
|
||||||
|
log.info(f"Piper model reloaded: {model_path}")
|
||||||
|
finally:
|
||||||
|
self._reloading.release()
|
||||||
|
|
||||||
|
def _model_path(self, *, warn: bool = False) -> Path | None:
|
||||||
|
"""The model to use (see the module docstring), or ``None`` if there is none."""
|
||||||
explicit = self.config.tts.piper_model.strip()
|
explicit = self.config.tts.piper_model.strip()
|
||||||
if explicit:
|
if explicit:
|
||||||
p = Path(explicit)
|
p = Path(explicit)
|
||||||
if p.exists():
|
if p.exists():
|
||||||
return p
|
return p
|
||||||
log.warning(f"piper_model path not found: {p}")
|
if warn:
|
||||||
|
log.warning(f"piper_model path not found: {p}")
|
||||||
|
|
||||||
# Fall back to active voice profile directory.
|
# Fall back to active voice profile directory.
|
||||||
|
candidate = (self.config.paths.voices_dir / self.config.tts.default_voice
|
||||||
|
/ "piper_model.onnx")
|
||||||
|
return candidate if candidate.exists() else None
|
||||||
|
|
||||||
|
def _resolve_model_path(self) -> Path:
|
||||||
|
"""Return the ONNX model path from config or voice profile dir."""
|
||||||
|
model_path = self._model_path(warn=True)
|
||||||
|
if model_path is not None:
|
||||||
|
return model_path
|
||||||
|
|
||||||
voice_name = self.config.tts.default_voice
|
voice_name = self.config.tts.default_voice
|
||||||
voices_dir = self.config.paths.voices_dir
|
voices_dir = self.config.paths.voices_dir
|
||||||
candidate = voices_dir / voice_name / "piper_model.onnx"
|
candidate = voices_dir / voice_name / "piper_model.onnx"
|
||||||
if candidate.exists():
|
|
||||||
return candidate
|
|
||||||
|
|
||||||
from ..voices.piper_train import has_checkpoints
|
from ..voices.piper_train import has_checkpoints
|
||||||
if has_checkpoints(voices_dir / voice_name):
|
if has_checkpoints(voices_dir / voice_name):
|
||||||
hint = (f"Training checkpoints exist — export them with: python -m "
|
hint = (f"Training checkpoints exist — export them with: python -m "
|
||||||
@@ -163,3 +245,16 @@ class PiperBackend(TTSBackend):
|
|||||||
f"Piper ONNX model not found. Expected at {candidate} "
|
f"Piper ONNX model not found. Expected at {candidate} "
|
||||||
f"or set tts.piper_model in config.yaml. {hint}"
|
f"or set tts.piper_model in config.yaml. {hint}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _files_signature(model_path: Path) -> tuple | None:
|
||||||
|
"""What tells a changed model apart: path, modification time and size of its files."""
|
||||||
|
try:
|
||||||
|
st = model_path.stat()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
config_mtime = Path(str(model_path) + ".json").stat().st_mtime_ns
|
||||||
|
except OSError:
|
||||||
|
config_mtime = None
|
||||||
|
return str(model_path), st.st_mtime_ns, st.st_size, config_mtime
|
||||||
@@ -30,6 +30,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field, asdict
|
from dataclasses import dataclass, field, asdict
|
||||||
from fractions import Fraction
|
from fractions import Fraction
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -106,6 +107,11 @@ _SEGMENTS_FILE = "ingest_segments.npy"
|
|||||||
# Source audio is transcoded once to a flat PCM WAV (mono, TARGET_SR) so that
|
# Source audio is transcoded once to a flat PCM WAV (mono, TARGET_SR) so that
|
||||||
# VAD and per-clip reads never decode compressed audio again.
|
# VAD and per-clip reads never decode compressed audio again.
|
||||||
_CACHE_RAW = "ingest_audio_cache.i16" # raw int16 LE, mono, TARGET_SR Hz
|
_CACHE_RAW = "ingest_audio_cache.i16" # raw int16 LE, mono, TARGET_SR Hz
|
||||||
|
# Free space that must remain on the cache drive; transcoding refuses to start
|
||||||
|
# or aborts mid-way below this.
|
||||||
|
_MIN_FREE_BYTES = 1024 * 1024 * 1024
|
||||||
|
# Re-check free space every N 10-second blocks (~10 min of audio, ~26 MB cache).
|
||||||
|
_DISK_CHECK_EVERY_BLOCKS = 60
|
||||||
# Save checkpoint every N processed segments (not iterations — skipped ones
|
# Save checkpoint every N processed segments (not iterations — skipped ones
|
||||||
# don't count) to limit how much work is lost on an unexpected stop.
|
# don't count) to limit how much work is lost on an unexpected stop.
|
||||||
_CHECKPOINT_SAVE_INTERVAL = 20
|
_CHECKPOINT_SAVE_INTERVAL = 20
|
||||||
@@ -328,55 +334,6 @@ def _is_clean_ref_clip(clip: np.ndarray, sr: int) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _load_audio(path: Path, target_sr: int = TARGET_SR) -> np.ndarray:
|
|
||||||
# soundfile 0.12+ ships libsndfile 1.2.2 which reads MP3 natively.
|
|
||||||
# OGG Vorbis and FLAC were always supported.
|
|
||||||
# M4A / AAC / OPUS / WMA are not supported by libsndfile — convert to
|
|
||||||
# MP3 or WAV first.
|
|
||||||
import soundfile as sf # type: ignore[import]
|
|
||||||
|
|
||||||
info = sf.info(str(path))
|
|
||||||
sr = info.samplerate
|
|
||||||
|
|
||||||
# MP3 files with malformed ID3 tags (e.g. non-syncsafe CTOC frames) cause
|
|
||||||
# libsndfile to report INT64_MAX for the frame count. Calling sf.read()
|
|
||||||
# then tries to pre-allocate that many samples and raises a ValueError.
|
|
||||||
# Fall back to chunked streaming via sf.blocks() which reads until EOF
|
|
||||||
# without pre-allocating based on the (bogus) total-frame count.
|
|
||||||
#
|
|
||||||
# Memory note: we use a small blocksize (10 s) and resample + mono-mix
|
|
||||||
# each block immediately so that only the TARGET_SR data accumulates
|
|
||||||
# (roughly 2× smaller than keeping raw source-rate stereo chunks).
|
|
||||||
if info.frames >= sys.maxsize // 2:
|
|
||||||
log.warning(f"_load_audio: unknown frame count for {path.name} — using chunked read")
|
|
||||||
frac = Fraction(target_sr, sr).limit_denominator(1000)
|
|
||||||
resampled_chunks: list[np.ndarray] = []
|
|
||||||
for block in sf.blocks(str(path), blocksize=sr * 10, dtype="float32", always_2d=True):
|
|
||||||
# Mono-mix immediately so we don't carry stereo data any longer than needed.
|
|
||||||
mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
|
|
||||||
chunk = (
|
|
||||||
scipy.signal.resample_poly(mono, frac.numerator, frac.denominator).astype(np.float32)
|
|
||||||
if sr != target_sr else mono.astype(np.float32)
|
|
||||||
)
|
|
||||||
resampled_chunks.append(chunk)
|
|
||||||
del block, mono # free source-rate data immediately
|
|
||||||
if not resampled_chunks:
|
|
||||||
raise ValueError(f"No audio data read from {path}")
|
|
||||||
audio: np.ndarray = np.concatenate(resampled_chunks)
|
|
||||||
del resampled_chunks
|
|
||||||
return audio # already mono, already at target_sr
|
|
||||||
else:
|
|
||||||
audio, _ = sf.read(str(path), dtype="float32", always_2d=True)
|
|
||||||
|
|
||||||
if audio.ndim > 1:
|
|
||||||
audio = audio.mean(axis=1)
|
|
||||||
|
|
||||||
if sr != target_sr:
|
|
||||||
frac = Fraction(target_sr, sr).limit_denominator(1000)
|
|
||||||
audio = scipy.signal.resample_poly(audio, frac.numerator, frac.denominator).astype(np.float32)
|
|
||||||
return audio
|
|
||||||
|
|
||||||
|
|
||||||
def _segment(audio: np.ndarray, sr: int) -> list[tuple[int, int]]:
|
def _segment(audio: np.ndarray, sr: int) -> list[tuple[int, int]]:
|
||||||
"""Return list of (start_sample, end_sample) speech segments via silero-vad."""
|
"""Return list of (start_sample, end_sample) speech segments via silero-vad."""
|
||||||
from silero_vad import load_silero_vad, get_speech_timestamps # heavy optional dep
|
from silero_vad import load_silero_vad, get_speech_timestamps # heavy optional dep
|
||||||
@@ -402,11 +359,21 @@ def _save_wav(path: Path, audio: np.ndarray, sr: int) -> None:
|
|||||||
sf.write(str(path), audio, sr, subtype="PCM_16")
|
sf.write(str(path), audio, sr, subtype="PCM_16")
|
||||||
|
|
||||||
|
|
||||||
def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
|
class TranscodeCancelled(Exception):
|
||||||
|
"""Raised from the transcode thread when the caller requested a stop."""
|
||||||
|
|
||||||
|
|
||||||
|
def _transcode_to_raw(
|
||||||
|
source: Path,
|
||||||
|
target: Path,
|
||||||
|
target_sr: int,
|
||||||
|
should_stop: Callable[[], bool] | None = None,
|
||||||
|
) -> None:
|
||||||
"""Stream *source* → raw int16 LE binary (mono, *target_sr* Hz).
|
"""Stream *source* → raw int16 LE binary (mono, *target_sr* Hz).
|
||||||
|
|
||||||
Peak RAM: one 10-second block (~4 MiB). No file-size limit.
|
Peak RAM: one 10-second block (~4 MiB). Writes atomically via a .part
|
||||||
Writes atomically via a .part temp file.
|
temp file, re-checks free disk space while writing and raises
|
||||||
|
:class:`TranscodeCancelled` as soon as *should_stop* returns True.
|
||||||
"""
|
"""
|
||||||
import soundfile as sf
|
import soundfile as sf
|
||||||
|
|
||||||
@@ -417,11 +384,11 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
|
|||||||
tmp = target.with_suffix(".part")
|
tmp = target.with_suffix(".part")
|
||||||
|
|
||||||
# Pre-check disk space when duration is known (skip for malformed MP3s with
|
# Pre-check disk space when duration is known (skip for malformed MP3s with
|
||||||
# bogus INT64_MAX frame counts).
|
# bogus INT64_MAX frame counts — the in-loop check below still applies).
|
||||||
if info.frames < sys.maxsize // 2:
|
if info.frames < sys.maxsize // 2:
|
||||||
estimated_bytes = int(info.frames / sr * target_sr) * 2 # int16 = 2 B
|
estimated_bytes = int(info.frames / sr * target_sr) * 2 # int16 = 2 B
|
||||||
free = shutil.disk_usage(str(target.parent)).free
|
free = shutil.disk_usage(str(target.parent)).free
|
||||||
if free < estimated_bytes + 64 * 1024 * 1024:
|
if free < estimated_bytes + _MIN_FREE_BYTES:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Not enough disk space to cache ‘{source.name}’: "
|
f"Not enough disk space to cache ‘{source.name}’: "
|
||||||
f"need ~{estimated_bytes / 1_000_000_000:.1f} GB, "
|
f"need ~{estimated_bytes / 1_000_000_000:.1f} GB, "
|
||||||
@@ -429,9 +396,18 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(str(tmp), "wb") as out_f:
|
# Read until SoundFile.read() returns 0 frames instead of using
|
||||||
for block in sf.blocks(str(source), blocksize=sr * 10,
|
# sf.blocks(): MP3s with malformed ID3 tags report INT64_MAX frames,
|
||||||
dtype="float32", always_2d=True):
|
# and sf.blocks() ignores the real read count, so after EOF it keeps
|
||||||
|
# re-yielding its stale buffer forever (filled 800 GB once).
|
||||||
|
with sf.SoundFile(str(source)) as src, open(str(tmp), "wb") as out_f:
|
||||||
|
n_blocks = 0
|
||||||
|
while True:
|
||||||
|
if should_stop is not None and should_stop():
|
||||||
|
raise TranscodeCancelled(source.name)
|
||||||
|
block = src.read(sr * 10, dtype="float32", always_2d=True)
|
||||||
|
if len(block) == 0:
|
||||||
|
break
|
||||||
mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
|
mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
|
||||||
if sr != target_sr:
|
if sr != target_sr:
|
||||||
chunk = scipy.signal.resample_poly(
|
chunk = scipy.signal.resample_poly(
|
||||||
@@ -442,6 +418,15 @@ def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
|
|||||||
i16 = np.clip(chunk * 32767.0, -32768, 32767).astype(np.int16)
|
i16 = np.clip(chunk * 32767.0, -32768, 32767).astype(np.int16)
|
||||||
out_f.write(i16.tobytes())
|
out_f.write(i16.tobytes())
|
||||||
del block, mono, chunk, i16
|
del block, mono, chunk, i16
|
||||||
|
n_blocks += 1
|
||||||
|
if n_blocks % _DISK_CHECK_EVERY_BLOCKS == 0:
|
||||||
|
free = shutil.disk_usage(str(target.parent)).free
|
||||||
|
if free < _MIN_FREE_BYTES:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Aborted caching ‘{source.name}’: free space on "
|
||||||
|
f"{target.parent} dropped to {free / 1_000_000_000:.1f} GB "
|
||||||
|
f"after writing {out_f.tell() / 1_000_000_000:.1f} GB"
|
||||||
|
)
|
||||||
tmp.replace(target)
|
tmp.replace(target)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
try:
|
||||||
@@ -599,6 +584,14 @@ async def ingest_audiobook(
|
|||||||
# peak RAM per block, no file-size limit). VAD and clip reads operate on
|
# peak RAM per block, no file-size limit). VAD and clip reads operate on
|
||||||
# this cache via numpy memmap — O(1) seek, ~90 MiB peak during segmentation.
|
# this cache via numpy memmap — O(1) seek, ~90 MiB peak during segmentation.
|
||||||
cache_raw = prof.dir / _CACHE_RAW
|
cache_raw = prof.dir / _CACHE_RAW
|
||||||
|
should_stop = stop_event.is_set if stop_event is not None else None
|
||||||
|
|
||||||
|
async def _stopped_while_transcoding() -> IngestStats:
|
||||||
|
log.info(f"ingest stopped by user while transcoding {source.name}")
|
||||||
|
await _emit({"phase": "stopped", "during": "transcoding",
|
||||||
|
"at_index": 0, "total": 0})
|
||||||
|
return stats
|
||||||
|
|
||||||
if cp is None:
|
if cp is None:
|
||||||
# Fresh run — remove any stale cache from a previous source.
|
# Fresh run — remove any stale cache from a previous source.
|
||||||
if cache_raw.exists():
|
if cache_raw.exists():
|
||||||
@@ -608,7 +601,11 @@ async def ingest_audiobook(
|
|||||||
pass
|
pass
|
||||||
log.info(f"transcoding {source.name} → raw cache ...")
|
log.info(f"transcoding {source.name} → raw cache ...")
|
||||||
await _emit({"phase": "loading", "source": source.name})
|
await _emit({"phase": "loading", "source": source.name})
|
||||||
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR)
|
try:
|
||||||
|
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR,
|
||||||
|
should_stop)
|
||||||
|
except TranscodeCancelled:
|
||||||
|
return await _stopped_while_transcoding()
|
||||||
_cache_dur = cache_raw.stat().st_size / 2 / TARGET_SR # int16 = 2 bytes
|
_cache_dur = cache_raw.stat().st_size / 2 / TARGET_SR # int16 = 2 bytes
|
||||||
log.info(f"transcoded: {_cache_dur:.1f}s")
|
log.info(f"transcoded: {_cache_dur:.1f}s")
|
||||||
await _emit({"phase": "loaded", "audio_s": _cache_dur})
|
await _emit({"phase": "loaded", "audio_s": _cache_dur})
|
||||||
@@ -616,7 +613,11 @@ async def ingest_audiobook(
|
|||||||
# Resuming but cache was deleted — retranscode.
|
# Resuming but cache was deleted — retranscode.
|
||||||
log.info(f"raw cache missing — retranscoding {source.name} ...")
|
log.info(f"raw cache missing — retranscoding {source.name} ...")
|
||||||
await _emit({"phase": "loading", "source": source.name})
|
await _emit({"phase": "loading", "source": source.name})
|
||||||
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR)
|
try:
|
||||||
|
await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR,
|
||||||
|
should_stop)
|
||||||
|
except TranscodeCancelled:
|
||||||
|
return await _stopped_while_transcoding()
|
||||||
log.info("retranscoded OK")
|
log.info("retranscoded OK")
|
||||||
|
|
||||||
if cp is not None:
|
if cp is not None:
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
+856
-204
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,495 @@
|
|||||||
|
"""Coordination between Javis and a running piper training process.
|
||||||
|
|
||||||
|
Shared by the launcher (:mod:`javis.voices.piper_train`, the overnight
|
||||||
|
scheduler, the GUI) and the training process (:mod:`javis.voices.piper_compat`
|
||||||
|
and its data-loader workers), so it imports nothing but the standard library.
|
||||||
|
|
||||||
|
* **Clean stops.** The launcher names a stop file in ``JAVIS_PIPER_STOP_FILE``;
|
||||||
|
once the file exists the trainer finishes its current step, saves its
|
||||||
|
progress — the epoch so far included — and exits with :data:`EXIT_STOPPED`.
|
||||||
|
It does the same if the launcher (``JAVIS_PIPER_LAUNCHER``) has gone away,
|
||||||
|
instead of training on unsupervised.
|
||||||
|
* **Closing the console window** (or logging off, shutting down) gives the
|
||||||
|
trainer the few seconds Windows allows to save its progress:
|
||||||
|
:class:`ConsoleClose`. :func:`python_command` starts it so that none of them
|
||||||
|
is lost: without the virtual environment's ``python.exe`` launcher, which
|
||||||
|
would end it at once, and without torch's Intel Fortran console handler.
|
||||||
|
* **One run per voice.** :class:`TrainingLock` records the processes of the run
|
||||||
|
holding it, so a crashed or killed run never blocks the next one.
|
||||||
|
* **Long runs on Windows.** :func:`keep_system_awake` and
|
||||||
|
:func:`console_selection_cannot_pause`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import functools
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
STOP_FILE_ENV = "JAVIS_PIPER_STOP_FILE"
|
||||||
|
LAUNCHER_ENV = "JAVIS_PIPER_LAUNCHER" # "<pid>:<start time>" of the supervising process
|
||||||
|
|
||||||
|
# Exit codes of ``python -m javis.voices.piper_compat fit`` and of run_pipeline().
|
||||||
|
EXIT_DIVERGED = 3 # training kept producing NaN/inf: restarting would not help
|
||||||
|
EXIT_STOPPED = 4 # stopped on request, after saving a checkpoint
|
||||||
|
EXIT_BUSY = 5 # run_pipeline(): another run of this voice holds the lock
|
||||||
|
|
||||||
|
LOCK_FILE = ".training.lock"
|
||||||
|
_LEGACY_LOCK_SECONDS = 48 * 60 * 60 # locks without processes (older Javis versions)
|
||||||
|
_UNREADABLE_LOCK_SECONDS = 60 # a lock file that is being written
|
||||||
|
|
||||||
|
PARTIAL_SUFFIX = ".partial" # a file being written; renamed into place once complete
|
||||||
|
# Windows: the target is open in another process for a moment (a snapshot export
|
||||||
|
# reading last.ckpt, a virus scanner): 32/33 sharing/lock violation, 1224 mapped.
|
||||||
|
_BUSY_WINERRORS = (32, 33, 1224)
|
||||||
|
|
||||||
|
|
||||||
|
def replace_file(src, dst, *, seconds: float = 60.0) -> None:
|
||||||
|
"""``os.replace`` that gives other processes up to *seconds* to close *dst*."""
|
||||||
|
delay, deadline = 0.1, time.monotonic() + seconds
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
os.replace(src, dst)
|
||||||
|
return
|
||||||
|
except OSError as exc:
|
||||||
|
if not (isinstance(exc, PermissionError)
|
||||||
|
or getattr(exc, "winerror", None) in _BUSY_WINERRORS) \
|
||||||
|
or time.monotonic() >= deadline:
|
||||||
|
raise
|
||||||
|
time.sleep(delay)
|
||||||
|
delay = min(delay * 2, 2.0)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Processes ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@functools.cache
|
||||||
|
def _kernel32():
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
# A private instance: argtypes set on ctypes.windll.kernel32 would leak to other code.
|
||||||
|
k = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||||
|
k.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD)
|
||||||
|
k.OpenProcess.restype = wintypes.HANDLE
|
||||||
|
k.GetExitCodeProcess.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD))
|
||||||
|
k.GetExitCodeProcess.restype = wintypes.BOOL
|
||||||
|
k.GetProcessTimes.argtypes = (wintypes.HANDLE,) + (ctypes.POINTER(wintypes.FILETIME),) * 4
|
||||||
|
k.GetProcessTimes.restype = wintypes.BOOL
|
||||||
|
k.CloseHandle.argtypes = (wintypes.HANDLE,)
|
||||||
|
k.CloseHandle.restype = wintypes.BOOL
|
||||||
|
k.SetThreadExecutionState.argtypes = (wintypes.DWORD,)
|
||||||
|
k.SetThreadExecutionState.restype = wintypes.DWORD
|
||||||
|
k.GetStdHandle.argtypes = (wintypes.DWORD,)
|
||||||
|
k.GetStdHandle.restype = wintypes.HANDLE
|
||||||
|
k.GetConsoleMode.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD))
|
||||||
|
k.GetConsoleMode.restype = wintypes.BOOL
|
||||||
|
k.SetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.DWORD)
|
||||||
|
k.SetConsoleMode.restype = wintypes.BOOL
|
||||||
|
k.SetConsoleCtrlHandler.argtypes = (ctypes.c_void_p, wintypes.BOOL)
|
||||||
|
k.SetConsoleCtrlHandler.restype = wintypes.BOOL
|
||||||
|
k.SetProcessShutdownParameters.argtypes = (wintypes.DWORD, wintypes.DWORD)
|
||||||
|
k.SetProcessShutdownParameters.restype = wintypes.BOOL
|
||||||
|
k.GetModuleHandleW.argtypes = (wintypes.LPCWSTR,)
|
||||||
|
k.GetModuleHandleW.restype = wintypes.HMODULE
|
||||||
|
return k
|
||||||
|
|
||||||
|
|
||||||
|
@functools.cache
|
||||||
|
def _user32():
|
||||||
|
"""``(user32, WNDPROC, WNDCLASSW)``: what a hidden window takes."""
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
wndproc = ctypes.WINFUNCTYPE(wintypes.LPARAM, wintypes.HWND, wintypes.UINT,
|
||||||
|
wintypes.WPARAM, wintypes.LPARAM)
|
||||||
|
|
||||||
|
class WNDCLASSW(ctypes.Structure):
|
||||||
|
_fields_ = [("style", wintypes.UINT), ("lpfnWndProc", wndproc),
|
||||||
|
("cbClsExtra", ctypes.c_int), ("cbWndExtra", ctypes.c_int),
|
||||||
|
("hInstance", wintypes.HINSTANCE), ("hIcon", wintypes.HICON),
|
||||||
|
("hCursor", wintypes.HANDLE), ("hbrBackground", wintypes.HBRUSH),
|
||||||
|
("lpszMenuName", wintypes.LPCWSTR), ("lpszClassName", wintypes.LPCWSTR)]
|
||||||
|
|
||||||
|
u = ctypes.WinDLL("user32", use_last_error=True)
|
||||||
|
u.RegisterClassW.argtypes = (ctypes.POINTER(WNDCLASSW),)
|
||||||
|
u.RegisterClassW.restype = wintypes.ATOM
|
||||||
|
u.UnregisterClassW.argtypes = (wintypes.LPCWSTR, wintypes.HINSTANCE)
|
||||||
|
u.UnregisterClassW.restype = wintypes.BOOL
|
||||||
|
u.CreateWindowExW.argtypes = (wintypes.DWORD, wintypes.LPCWSTR, wintypes.LPCWSTR,
|
||||||
|
wintypes.DWORD, ctypes.c_int, ctypes.c_int, ctypes.c_int,
|
||||||
|
ctypes.c_int, wintypes.HWND, wintypes.HMENU,
|
||||||
|
wintypes.HINSTANCE, wintypes.LPVOID)
|
||||||
|
u.CreateWindowExW.restype = wintypes.HWND
|
||||||
|
u.DefWindowProcW.argtypes = (wintypes.HWND, wintypes.UINT, wintypes.WPARAM,
|
||||||
|
wintypes.LPARAM)
|
||||||
|
u.DefWindowProcW.restype = wintypes.LPARAM
|
||||||
|
u.GetMessageW.argtypes = (ctypes.POINTER(wintypes.MSG), wintypes.HWND, wintypes.UINT,
|
||||||
|
wintypes.UINT)
|
||||||
|
u.GetMessageW.restype = wintypes.BOOL
|
||||||
|
u.DispatchMessageW.argtypes = (ctypes.POINTER(wintypes.MSG),)
|
||||||
|
u.DispatchMessageW.restype = wintypes.LPARAM
|
||||||
|
u.PostQuitMessage.argtypes = (ctypes.c_int,)
|
||||||
|
u.PostQuitMessage.restype = None
|
||||||
|
return u, wndproc, WNDCLASSW
|
||||||
|
|
||||||
|
|
||||||
|
def python_command() -> tuple[str, dict]:
|
||||||
|
"""``(executable, extra environment)`` to start another Python of this environment.
|
||||||
|
|
||||||
|
On Windows a virtual environment's ``python.exe`` is a launcher running the
|
||||||
|
real interpreter in a job that dies with it — and the launcher ends as soon
|
||||||
|
as its console window closes, so the interpreter couldn't save anything.
|
||||||
|
Like :mod:`multiprocessing`, start the real interpreter directly;
|
||||||
|
``__PYVENV_LAUNCHER__`` tells it which environment it belongs to.
|
||||||
|
|
||||||
|
Nor does it get the console handler of the Intel Fortran runtime in torch's
|
||||||
|
``libiomp5md.dll``: on a closing window that aborts every process loading
|
||||||
|
torch ("forrtl: error (200)"), and slowly — Windows tells the processes of a
|
||||||
|
console one after the other, the newest first: data-loader workers and
|
||||||
|
exports before the trainer, which had seconds less to save its progress.
|
||||||
|
"""
|
||||||
|
if sys.platform != "win32":
|
||||||
|
return sys.executable, {}
|
||||||
|
env = {"FOR_DISABLE_CONSOLE_CTRL_HANDLER": "1"} # inherited by data-loader workers
|
||||||
|
base = getattr(sys, "_base_executable", "") or ""
|
||||||
|
if base and os.path.isfile(base) and os.path.normcase(os.path.abspath(base)) \
|
||||||
|
!= os.path.normcase(os.path.abspath(sys.executable)):
|
||||||
|
return base, {**env, "__PYVENV_LAUNCHER__": sys.executable}
|
||||||
|
return sys.executable, env
|
||||||
|
|
||||||
|
|
||||||
|
def process_start_time(pid: int) -> int | None:
|
||||||
|
"""When process *pid* started, as an opaque number; ``None`` if it isn't running.
|
||||||
|
|
||||||
|
``0`` means it runs but its start time can't be read.
|
||||||
|
"""
|
||||||
|
if pid <= 0:
|
||||||
|
return None
|
||||||
|
if sys.platform != "win32":
|
||||||
|
try:
|
||||||
|
os.kill(pid, 0) # signal 0 only checks; on Windows os.kill would terminate
|
||||||
|
except PermissionError:
|
||||||
|
return 0
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
return 0
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
k = _kernel32()
|
||||||
|
handle = k.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
|
||||||
|
if not handle:
|
||||||
|
return 0 if ctypes.get_last_error() == 5 else None # 5: access denied, yet it exists
|
||||||
|
try:
|
||||||
|
code = wintypes.DWORD()
|
||||||
|
if k.GetExitCodeProcess(handle, ctypes.byref(code)) and code.value != 259: # STILL_ACTIVE
|
||||||
|
return None
|
||||||
|
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
|
||||||
|
if not k.GetProcessTimes(handle, ctypes.byref(created), ctypes.byref(exited),
|
||||||
|
ctypes.byref(kernel), ctypes.byref(user)):
|
||||||
|
return 0
|
||||||
|
return (created.dwHighDateTime << 32) | created.dwLowDateTime
|
||||||
|
finally:
|
||||||
|
k.CloseHandle(handle)
|
||||||
|
|
||||||
|
|
||||||
|
def process_alive(pid: int, start_time: int | None = None) -> bool:
|
||||||
|
"""Whether *pid* is running and, given its *start_time*, is still the same process.
|
||||||
|
|
||||||
|
Windows soon hands the ID of an exited process to a new one.
|
||||||
|
"""
|
||||||
|
current = process_start_time(pid)
|
||||||
|
if current is None:
|
||||||
|
return False
|
||||||
|
return not start_time or not current or current == start_time
|
||||||
|
|
||||||
|
|
||||||
|
def this_process() -> dict:
|
||||||
|
"""``{"pid", "start"}`` of the calling process."""
|
||||||
|
pid = os.getpid()
|
||||||
|
return {"pid": pid, "start": process_start_time(pid) or 0}
|
||||||
|
|
||||||
|
|
||||||
|
def launcher_from_environment() -> tuple[int, int] | None:
|
||||||
|
"""``(pid, start time)`` of the launcher named in ``JAVIS_PIPER_LAUNCHER``, if any."""
|
||||||
|
value = os.environ.get(LAUNCHER_ENV, "")
|
||||||
|
try:
|
||||||
|
pid, start = (int(part) for part in value.split(":"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return pid, start
|
||||||
|
|
||||||
|
|
||||||
|
# ── One training run per voice ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def lock_path(voice_dir: Path) -> Path:
|
||||||
|
return Path(voice_dir) / "training" / LOCK_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def _read_lock(path: Path) -> str | None:
|
||||||
|
"""The lock file's text; ``None`` if there is none. Raises other ``OSError``s."""
|
||||||
|
try:
|
||||||
|
return path.read_text(encoding="utf-8")
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _held(path: Path, text: str) -> bool:
|
||||||
|
try:
|
||||||
|
age = time.time() - path.stat().st_mtime
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
except OSError:
|
||||||
|
return True # can't tell: assume it is held
|
||||||
|
try:
|
||||||
|
info = json.loads(text)
|
||||||
|
except ValueError:
|
||||||
|
info = None
|
||||||
|
if not isinstance(info, dict):
|
||||||
|
return age < _UNREADABLE_LOCK_SECONDS
|
||||||
|
try:
|
||||||
|
processes = [(int(p.get("pid") or 0), int(p.get("start") or 0))
|
||||||
|
for p in info.get("processes") or () if isinstance(p, dict)]
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
processes = []
|
||||||
|
if not processes or info.get("host", platform.node()) != platform.node():
|
||||||
|
try:
|
||||||
|
started = float(info.get("started_epoch") or 0) or path.stat().st_mtime
|
||||||
|
except (OSError, TypeError, ValueError):
|
||||||
|
return True
|
||||||
|
return time.time() - started < _LEGACY_LOCK_SECONDS
|
||||||
|
return any(process_alive(pid, start) for pid, start in processes)
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_held(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
text = _read_lock(path)
|
||||||
|
except OSError:
|
||||||
|
return True
|
||||||
|
return text is not None and _held(path, text)
|
||||||
|
|
||||||
|
|
||||||
|
def training_in_progress(voice_dir: Path) -> bool:
|
||||||
|
"""Whether a training run of this voice is running (in any process)."""
|
||||||
|
return _lock_held(lock_path(voice_dir))
|
||||||
|
|
||||||
|
|
||||||
|
class TrainingLock:
|
||||||
|
"""``training/.training.lock``: one training run per voice at a time, across processes.
|
||||||
|
|
||||||
|
Lists the launcher and, while one runs, its training process (which keeps
|
||||||
|
training for a minute or two after its launcher was killed, to save its
|
||||||
|
progress). A lock whose processes have all exited is stale and replaced.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, voice_dir: Path) -> None:
|
||||||
|
self.path = lock_path(voice_dir)
|
||||||
|
self._info: dict | None = None
|
||||||
|
|
||||||
|
def acquire(self) -> bool:
|
||||||
|
"""Take the lock; ``False`` if another run holds it."""
|
||||||
|
info = {"host": platform.node(), "started_epoch": time.time(),
|
||||||
|
"processes": [this_process()]}
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
for _ in range(2):
|
||||||
|
try:
|
||||||
|
with open(self.path, "x", encoding="utf-8") as f:
|
||||||
|
json.dump(info, f)
|
||||||
|
except FileExistsError:
|
||||||
|
try:
|
||||||
|
text = _read_lock(self.path)
|
||||||
|
if text is not None:
|
||||||
|
if _held(self.path, text):
|
||||||
|
return False
|
||||||
|
# Left behind by a run that crashed or was killed — unless
|
||||||
|
# another launcher replaced it just now.
|
||||||
|
if _read_lock(self.path) == text:
|
||||||
|
self.path.unlink()
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
continue
|
||||||
|
self._info = info
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def set_trainer(self, pid: int | None) -> None:
|
||||||
|
"""Record the training process (or that none runs right now)."""
|
||||||
|
if self._info is None:
|
||||||
|
return
|
||||||
|
trainer = [{"pid": pid, "start": process_start_time(pid) or 0}] if pid else []
|
||||||
|
self._info["processes"] = self._info["processes"][:1] + trainer
|
||||||
|
with contextlib.suppress(OSError):
|
||||||
|
self.path.write_text(json.dumps(self._info), encoding="utf-8")
|
||||||
|
|
||||||
|
def release(self) -> None:
|
||||||
|
if self._info is None:
|
||||||
|
return
|
||||||
|
owner, self._info = self._info["processes"][0], None
|
||||||
|
try:
|
||||||
|
current = json.loads(self.path.read_text(encoding="utf-8"))
|
||||||
|
if (current.get("processes") or [{}])[0] == owner:
|
||||||
|
self.path.unlink()
|
||||||
|
except (OSError, ValueError, AttributeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# ── Windows ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def keep_system_awake():
|
||||||
|
"""Keep Windows from sleeping when idle (the display may still turn off)."""
|
||||||
|
if sys.platform != "win32":
|
||||||
|
yield
|
||||||
|
return
|
||||||
|
k = _kernel32()
|
||||||
|
es_continuous, es_system_required = 0x80000000, 0x00000001
|
||||||
|
previous = k.SetThreadExecutionState(es_continuous | es_system_required)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
k.SetThreadExecutionState(previous if previous & es_continuous else es_continuous)
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def console_selection_cannot_pause():
|
||||||
|
"""Turn off QuickEdit in a classic console window while training runs.
|
||||||
|
|
||||||
|
There, clicking into the window starts a selection that suspends every
|
||||||
|
process writing to it — training would stall until a key is pressed.
|
||||||
|
(Windows Terminal doesn't do that.)
|
||||||
|
"""
|
||||||
|
if sys.platform != "win32":
|
||||||
|
yield
|
||||||
|
return
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
k = _kernel32()
|
||||||
|
handle = k.GetStdHandle(wintypes.DWORD(-10).value) # STD_INPUT_HANDLE
|
||||||
|
mode = wintypes.DWORD()
|
||||||
|
quick_edit, extended_flags = 0x0040, 0x0080
|
||||||
|
if not handle or not k.GetConsoleMode(handle, ctypes.byref(mode)) \
|
||||||
|
or not mode.value & quick_edit:
|
||||||
|
yield
|
||||||
|
return
|
||||||
|
k.SetConsoleMode(handle, (mode.value | extended_flags) & ~quick_edit)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
k.SetConsoleMode(handle, mode.value | extended_flags)
|
||||||
|
|
||||||
|
|
||||||
|
_CONSOLE_CLOSE_EVENTS = {2: "the console window was closed", 5: "the user is logging off",
|
||||||
|
6: "the system is shutting down"}
|
||||||
|
_WM_DESTROY, _WM_QUERYENDSESSION, _WM_ENDSESSION = 0x0002, 0x0011, 0x0016
|
||||||
|
_ENDSESSION_LOGOFF = 0x80000000
|
||||||
|
# Ends before the other processes of the session — its data-loader workers
|
||||||
|
# among them: the top of the range Windows leaves to applications.
|
||||||
|
_SHUTDOWN_LEVEL = 0x3FF
|
||||||
|
|
||||||
|
|
||||||
|
class ConsoleClose:
|
||||||
|
"""Lets the training process save its progress before Windows ends it.
|
||||||
|
|
||||||
|
When its console window is closed (or the user logs off, the system shuts
|
||||||
|
down), Windows ends every process attached to it after a few seconds.
|
||||||
|
:meth:`install` handles those events: it sets :attr:`closing` — the training
|
||||||
|
loop checks it after every step — and holds the process until :meth:`done`
|
||||||
|
reports the progress saved, or *timeout* seconds have passed.
|
||||||
|
|
||||||
|
Logging off and shutting down reach a console handler only in processes
|
||||||
|
that never loaded ``user32.dll`` — Python and torch load it. A hidden
|
||||||
|
window learns of them instead (``WM_ENDSESSION``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, timeout: float = 30.0) -> None:
|
||||||
|
self.closing = threading.Event()
|
||||||
|
self.reason = ""
|
||||||
|
self.timeout = timeout
|
||||||
|
self._saved = threading.Event()
|
||||||
|
self._handler = None
|
||||||
|
self._window = None # the hidden window's handle
|
||||||
|
|
||||||
|
def handle(self, event: int) -> bool:
|
||||||
|
"""The console control handler; runs in a thread of its own."""
|
||||||
|
reason = _CONSOLE_CLOSE_EVENTS.get(event)
|
||||||
|
if reason is None:
|
||||||
|
return False # Ctrl+C, Ctrl+Break: left to Python's own handler
|
||||||
|
self._close(reason)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _close(self, reason: str) -> None:
|
||||||
|
self.reason = reason
|
||||||
|
self.closing.set()
|
||||||
|
self._saved.wait(self.timeout)
|
||||||
|
|
||||||
|
def done(self) -> None:
|
||||||
|
"""The progress is saved: Windows may end the process now."""
|
||||||
|
self._saved.set()
|
||||||
|
|
||||||
|
def install(self) -> None:
|
||||||
|
if sys.platform != "win32" or self._handler is not None:
|
||||||
|
return
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
handler = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD)(self.handle)
|
||||||
|
if _kernel32().SetConsoleCtrlHandler(handler, True):
|
||||||
|
self._handler = handler # must outlive every call Windows may make
|
||||||
|
_kernel32().SetProcessShutdownParameters(_SHUTDOWN_LEVEL, 0)
|
||||||
|
self._watch_session_end()
|
||||||
|
|
||||||
|
def _watch_session_end(self) -> None:
|
||||||
|
"""Open the hidden window that learns of the user logging off, the system shutting down."""
|
||||||
|
ready = threading.Event()
|
||||||
|
threading.Thread(target=self._session_window, args=(ready,), name="session end",
|
||||||
|
daemon=True).start()
|
||||||
|
ready.wait(10)
|
||||||
|
|
||||||
|
def _session_window(self, ready: threading.Event) -> None:
|
||||||
|
import ctypes
|
||||||
|
from ctypes import wintypes
|
||||||
|
|
||||||
|
try:
|
||||||
|
user32, wndproc, wndclass = _user32()
|
||||||
|
instance = _kernel32().GetModuleHandleW(None)
|
||||||
|
|
||||||
|
def window_procedure(hwnd, message, wparam, lparam):
|
||||||
|
if message == _WM_QUERYENDSESSION:
|
||||||
|
return 1 # may end: the progress gets saved at WM_ENDSESSION
|
||||||
|
if message == _WM_ENDSESSION:
|
||||||
|
if wparam: # else cancelled
|
||||||
|
self._close("the user is logging off" if lparam & _ENDSESSION_LOGOFF
|
||||||
|
else "the system is shutting down")
|
||||||
|
return 0
|
||||||
|
if message == _WM_DESTROY:
|
||||||
|
user32.PostQuitMessage(0)
|
||||||
|
return 0
|
||||||
|
return user32.DefWindowProcW(hwnd, message, wparam, lparam)
|
||||||
|
|
||||||
|
procedure = wndproc(window_procedure) # referenced while the window exists
|
||||||
|
name = f"JavisTrainingSession{os.getpid()}-{id(self):x}"
|
||||||
|
if not user32.RegisterClassW(ctypes.byref(wndclass(
|
||||||
|
lpfnWndProc=procedure, hInstance=instance, lpszClassName=name))):
|
||||||
|
return
|
||||||
|
# Top-level but never shown: message-only windows get no WM_ENDSESSION.
|
||||||
|
window = user32.CreateWindowExW(0, name, "Javis voice training", 0, 0, 0, 0, 0,
|
||||||
|
None, None, instance, None)
|
||||||
|
if not window:
|
||||||
|
user32.UnregisterClassW(name, instance)
|
||||||
|
return
|
||||||
|
self._window = window
|
||||||
|
finally:
|
||||||
|
ready.set()
|
||||||
|
message = wintypes.MSG()
|
||||||
|
while user32.GetMessageW(ctypes.byref(message), None, 0, 0) > 0:
|
||||||
|
user32.DispatchMessageW(ctypes.byref(message))
|
||||||
|
user32.UnregisterClassW(name, instance)
|
||||||
@@ -18,31 +18,29 @@ Design notes
|
|||||||
without a restart.
|
without a restart.
|
||||||
* **Window guard.** Training will not *start* if fewer than
|
* **Window guard.** Training will not *start* if fewer than
|
||||||
``min_remaining_minutes`` of the window are left. A run that has already
|
``min_remaining_minutes`` of the window are left. A run that has already
|
||||||
started is allowed to run to completion — PyTorch-Lightning cannot be
|
started keeps going; quitting Javis stops it cleanly — the training
|
||||||
cleanly killed mid-epoch, and an unfinished run is safe: it leaves a
|
process saves its progress after its current step (the epoch so far
|
||||||
checkpoint that the next run resumes from.
|
included) first, so Javis's process can take a little while to exit —
|
||||||
* **Concurrency guard.** A lock file (``training/.training.lock``) prevents
|
and the next run resumes from where it stopped. While it runs, the
|
||||||
the scheduler and the interactive GUI button from training the same voice
|
voice's model is updated with its progress (see ``piper_train``).
|
||||||
at the same time (they run in separate processes).
|
* **Concurrency guard.** ``run_pipeline`` holds a lock file
|
||||||
|
(``training/.training.lock``) naming its processes, so the scheduler and
|
||||||
|
the interactive GUI button (separate processes) never train the same voice
|
||||||
|
at the same time; a run that crashed or was killed leaves no lasting lock.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
|
||||||
import time
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from ..core.config import Config
|
from ..core.config import Config
|
||||||
from ..core.logger import get_logger
|
from ..core.logger import get_logger
|
||||||
|
from .training_control import EXIT_BUSY, EXIT_STOPPED, training_in_progress
|
||||||
|
|
||||||
log = get_logger("voices.training")
|
log = get_logger("voices.training")
|
||||||
|
|
||||||
# A lock is considered "live" for this long. Overnight runs take a few
|
|
||||||
# hours; 48 h comfortably bounds a stuck/crashed run so it self-expires.
|
|
||||||
_LOCK_STALE_SECONDS = 48 * 60 * 60
|
|
||||||
|
|
||||||
|
|
||||||
# ── Pure helpers (unit-testable) ─────────────────────────────────────────────
|
# ── Pure helpers (unit-testable) ─────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -97,62 +95,10 @@ def window_end(now: datetime, start_hour: int, end_hour: int) -> datetime:
|
|||||||
return end
|
return end
|
||||||
|
|
||||||
|
|
||||||
# ── Cross-process lock ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
def _lock_path(voice_dir: Path) -> Path:
|
|
||||||
return voice_dir / "training" / ".training.lock"
|
|
||||||
|
|
||||||
|
|
||||||
def _lock_alive(voice_dir: Path) -> bool:
|
|
||||||
"""True if a recent lock file indicates another training is in progress."""
|
|
||||||
lp = _lock_path(voice_dir)
|
|
||||||
if not lp.exists():
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
data = json.loads(lp.read_text(encoding="utf-8"))
|
|
||||||
ts = float(data.get("started_epoch", 0.0))
|
|
||||||
except Exception:
|
|
||||||
try:
|
|
||||||
ts = lp.stat().st_mtime
|
|
||||||
except OSError:
|
|
||||||
return False
|
|
||||||
if not ts:
|
|
||||||
return False
|
|
||||||
return (time.time() - ts) < _LOCK_STALE_SECONDS
|
|
||||||
|
|
||||||
|
|
||||||
def _acquire_lock(voice_dir: Path) -> bool:
|
|
||||||
"""Atomically claim the training lock. Returns False if already held."""
|
|
||||||
lp = _lock_path(voice_dir)
|
|
||||||
lp.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
try:
|
|
||||||
# 'x' = exclusive create; fails if a live lock already exists.
|
|
||||||
with open(lp, "x", encoding="utf-8") as f:
|
|
||||||
f.write(json.dumps({"started_epoch": time.time()}))
|
|
||||||
return True
|
|
||||||
except FileExistsError:
|
|
||||||
return False
|
|
||||||
except OSError:
|
|
||||||
# On platforms where 'x' is flaky, fall back to the liveness check.
|
|
||||||
return not _lock_alive(voice_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def _release_lock(voice_dir: Path) -> None:
|
|
||||||
try:
|
|
||||||
_lock_path(voice_dir).unlink(missing_ok=True)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def training_in_progress(voice_dir: Path) -> bool:
|
|
||||||
"""Public helper: is another training of this voice currently running?"""
|
|
||||||
return _lock_alive(voice_dir)
|
|
||||||
|
|
||||||
|
|
||||||
# ── One run ───────────────────────────────────────────────────────────────────
|
# ── One run ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = None) -> None:
|
async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = None) -> None:
|
||||||
"""Execute a single scheduled training run (threaded) with locking."""
|
"""Execute a single scheduled training run (threaded)."""
|
||||||
from .piper_train import run_pipeline # imported lazily (heavy deps)
|
from .piper_train import run_pipeline # imported lazily (heavy deps)
|
||||||
|
|
||||||
t = config.training
|
t = config.training
|
||||||
@@ -166,14 +112,10 @@ async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = Non
|
|||||||
f"{t.min_remaining_minutes} min minimum); skipping start.")
|
f"{t.min_remaining_minutes} min minimum); skipping start.")
|
||||||
return
|
return
|
||||||
|
|
||||||
if _lock_alive(voice_dir):
|
if training_in_progress(voice_dir):
|
||||||
log.info("Another training of this voice is already running; skipping.")
|
log.info("Another training of this voice is already running; skipping.")
|
||||||
return
|
return
|
||||||
|
|
||||||
if not _acquire_lock(voice_dir):
|
|
||||||
log.info("Could not acquire training lock (already held); skipping.")
|
|
||||||
return
|
|
||||||
|
|
||||||
voice = voice_dir.name
|
voice = voice_dir.name
|
||||||
n_clips = 0
|
n_clips = 0
|
||||||
meta = voice_dir / "training" / "metadata.csv"
|
meta = voice_dir / "training" / "metadata.csv"
|
||||||
@@ -214,25 +156,30 @@ async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = Non
|
|||||||
try:
|
try:
|
||||||
rc = await asyncio.shield(work)
|
rc = await asyncio.shield(work)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
# Javis is quitting: stop the child process so the worker thread (and
|
# Javis is quitting: have the training process save a checkpoint and
|
||||||
# interpreter exit) doesn't block until training finishes.
|
# stop, so the worker thread (and interpreter exit) isn't kept waiting
|
||||||
from .piper_train import request_stop
|
# until training finishes.
|
||||||
|
from .piper_train import STOP_GRACE_SECONDS, request_stop
|
||||||
request_stop()
|
request_stop()
|
||||||
await asyncio.wait({work}, timeout=60)
|
await asyncio.wait({work}, timeout=STOP_GRACE_SECONDS + 30)
|
||||||
raise
|
raise
|
||||||
except Exception as exc: # noqa: BLE001 — training must never kill the loop
|
except Exception as exc: # noqa: BLE001 — training must never kill the loop
|
||||||
log.exception(f"Scheduled training crashed: {exc}")
|
log.exception(f"Scheduled training crashed: {exc}")
|
||||||
rc = -1
|
rc = -1
|
||||||
finally:
|
|
||||||
if work.done():
|
|
||||||
_release_lock(voice_dir)
|
|
||||||
|
|
||||||
status = "completed" if rc == 0 else f"FAILED (exit {rc})"
|
if rc == 0:
|
||||||
|
status = "completed"
|
||||||
|
elif rc == EXIT_STOPPED:
|
||||||
|
status = "stopped (progress saved)"
|
||||||
|
elif rc == EXIT_BUSY:
|
||||||
|
status = "skipped (this voice is being trained already)"
|
||||||
|
else:
|
||||||
|
status = f"FAILED (exit {rc})"
|
||||||
log.info(f"Scheduled training {status}: voice={voice!r}")
|
log.info(f"Scheduled training {status}: voice={voice!r}")
|
||||||
if bus is not None:
|
if bus is not None:
|
||||||
await _safe_publish(
|
await _safe_publish(
|
||||||
bus, "task_done",
|
bus, "task_done",
|
||||||
{"id": "training", "ok": rc == 0,
|
{"id": "training", "ok": rc in (0, EXIT_STOPPED, EXIT_BUSY),
|
||||||
"msg": f"Voice training {status}"})
|
"msg": f"Voice training {status}"})
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""Audiobook ingest: raw-cache transcoding must terminate and clean up."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from collections import namedtuple
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock, PropertyMock, patch
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
SRC_SR = 44100
|
||||||
|
SRC_SECONDS = 25 # two full 10 s blocks + one partial block
|
||||||
|
TARGET_SR = 22050
|
||||||
|
|
||||||
|
_DiskUsage = namedtuple("_DiskUsage", "total used free")
|
||||||
|
|
||||||
|
|
||||||
|
class TranscodeToRawTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
try:
|
||||||
|
import soundfile as sf
|
||||||
|
except ImportError:
|
||||||
|
self.skipTest("soundfile not installed")
|
||||||
|
from javis.voices import audiobook_ingest
|
||||||
|
self.sf = sf
|
||||||
|
self.ingest = audiobook_ingest
|
||||||
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self._tmp.name)
|
||||||
|
self.source = self.root / "book.wav"
|
||||||
|
t = np.arange(SRC_SR * SRC_SECONDS) / SRC_SR
|
||||||
|
tone = 0.3 * np.sin(2 * np.pi * 220 * t)
|
||||||
|
sf.write(str(self.source), np.stack([tone, tone], axis=1), SRC_SR, subtype="PCM_16")
|
||||||
|
self.target = self.root / "cache" / "ingest_audio_cache.i16"
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self._tmp.cleanup()
|
||||||
|
|
||||||
|
def _bogus_frame_count(self):
|
||||||
|
"""Mimic an MP3 with a malformed ID3 tag: INT64_MAX frames, unseekable."""
|
||||||
|
frames = patch.object(self.sf.SoundFile, "frames", new_callable=PropertyMock,
|
||||||
|
return_value=sys.maxsize)
|
||||||
|
seekable = patch.object(self.sf.SoundFile, "seekable", return_value=False)
|
||||||
|
return frames, seekable
|
||||||
|
|
||||||
|
def _run_with_timeout(self, timeout: float = 60.0, **kw) -> BaseException | None:
|
||||||
|
errors: list[BaseException] = []
|
||||||
|
|
||||||
|
def run():
|
||||||
|
try:
|
||||||
|
self.ingest._transcode_to_raw(self.source, self.target, TARGET_SR, **kw)
|
||||||
|
except BaseException as e: # noqa: BLE001
|
||||||
|
errors.append(e)
|
||||||
|
|
||||||
|
th = threading.Thread(target=run, daemon=True)
|
||||||
|
th.start()
|
||||||
|
th.join(timeout)
|
||||||
|
self.assertFalse(th.is_alive(), "transcode did not stop at EOF (runaway)")
|
||||||
|
return errors[0] if errors else None
|
||||||
|
|
||||||
|
def test_unknown_frame_count_stops_at_eof(self) -> None:
|
||||||
|
frames, seekable = self._bogus_frame_count()
|
||||||
|
with frames, seekable:
|
||||||
|
self.assertGreaterEqual(self.sf.info(str(self.source)).frames, sys.maxsize // 2)
|
||||||
|
err = self._run_with_timeout()
|
||||||
|
self.assertIsNone(err)
|
||||||
|
self.assertEqual(self.target.stat().st_size, SRC_SECONDS * TARGET_SR * 2)
|
||||||
|
self.assertFalse(self.target.with_suffix(".part").exists())
|
||||||
|
|
||||||
|
def test_stop_request_aborts_and_removes_partial_file(self) -> None:
|
||||||
|
err = self._run_with_timeout(should_stop=lambda: True)
|
||||||
|
self.assertIsInstance(err, self.ingest.TranscodeCancelled)
|
||||||
|
self.assertFalse(self.target.exists())
|
||||||
|
self.assertFalse(self.target.with_suffix(".part").exists())
|
||||||
|
|
||||||
|
def test_disk_running_full_aborts_and_removes_partial_file(self) -> None:
|
||||||
|
frames, seekable = self._bogus_frame_count()
|
||||||
|
full = patch.object(self.ingest.shutil, "disk_usage",
|
||||||
|
return_value=_DiskUsage(10**12, 10**12, 0))
|
||||||
|
every_block = patch.object(self.ingest, "_DISK_CHECK_EVERY_BLOCKS", 1)
|
||||||
|
with frames, seekable, full, every_block:
|
||||||
|
err = self._run_with_timeout()
|
||||||
|
self.assertIsInstance(err, RuntimeError)
|
||||||
|
self.assertIn("free space", str(err))
|
||||||
|
self.assertFalse(self.target.exists())
|
||||||
|
self.assertFalse(self.target.with_suffix(".part").exists())
|
||||||
|
|
||||||
|
def test_insufficient_space_for_known_length_fails_fast(self) -> None:
|
||||||
|
with patch.object(self.ingest.shutil, "disk_usage",
|
||||||
|
return_value=_DiskUsage(10**12, 10**12, 0)):
|
||||||
|
err = self._run_with_timeout()
|
||||||
|
self.assertIsInstance(err, RuntimeError)
|
||||||
|
self.assertIn("Not enough disk space", str(err))
|
||||||
|
self.assertFalse(self.target.with_suffix(".part").exists())
|
||||||
|
|
||||||
|
def test_stop_button_during_transcode_ends_ingest_as_stopped(self) -> None:
|
||||||
|
events: list[dict] = []
|
||||||
|
|
||||||
|
class Bus:
|
||||||
|
async def publish(self, topic, payload):
|
||||||
|
events.append(payload)
|
||||||
|
|
||||||
|
mgr = MagicMock()
|
||||||
|
mgr.root = self.root
|
||||||
|
mgr.create.return_value = SimpleNamespace(dir=self.root / "voice")
|
||||||
|
stop = asyncio.Event()
|
||||||
|
stop.set()
|
||||||
|
with patch("javis.voices.manager.VoiceManager", return_value=mgr):
|
||||||
|
stats = asyncio.run(self.ingest.ingest_audiobook(
|
||||||
|
None, self.source, "voice", bus=Bus(), stop_event=stop, resume=False))
|
||||||
|
self.assertEqual(stats.total_segments, 0)
|
||||||
|
self.assertEqual(events[-1]["phase"], "stopped")
|
||||||
|
self.assertEqual(events[-1]["during"], "transcoding")
|
||||||
|
self.assertFalse((self.root / "voice" / "ingest_audio_cache.i16").exists())
|
||||||
|
self.assertFalse((self.root / "voice" / "ingest_audio_cache.part").exists())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Slave GUI chat: newest-first transcript and fullscreen chat mode (offscreen Qt)."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
|
||||||
|
|
||||||
|
SLAVE_SRC = Path(__file__).resolve().parents[1] / "javis-slave" / "src"
|
||||||
|
if str(SLAVE_SRC) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SLAVE_SRC))
|
||||||
|
|
||||||
|
QtWidgets = pytest.importorskip("PySide6.QtWidgets")
|
||||||
|
QtTest = pytest.importorskip("PySide6.QtTest")
|
||||||
|
|
||||||
|
from javis_slave import main_window # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def qapp():
|
||||||
|
return QtWidgets.QApplication.instance() or QtWidgets.QApplication([])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def screen(qapp):
|
||||||
|
scr = main_window.MainScreen()
|
||||||
|
scr.resize(900, 700)
|
||||||
|
scr.show()
|
||||||
|
yield scr
|
||||||
|
scr.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _texts(scr) -> list[str]:
|
||||||
|
"""Bubble texts from the top of the transcript to the bottom."""
|
||||||
|
layout = scr.transcript_layout
|
||||||
|
out = []
|
||||||
|
for i in range(layout.count()):
|
||||||
|
row = layout.itemAt(i).widget()
|
||||||
|
if row is not None:
|
||||||
|
out.append(row.findChild(QtWidgets.QLabel).text())
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def test_newest_message_is_on_top(screen):
|
||||||
|
screen.set_turn("frage 1", "antwort 1")
|
||||||
|
screen.set_turn("frage 2", "antwort 2")
|
||||||
|
screen.show_error("kaputt")
|
||||||
|
assert _texts(screen) == [
|
||||||
|
"⚠ kaputt", "antwort 2", "frage 2", "antwort 1", "frage 1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_drops_oldest_at_the_bottom(screen):
|
||||||
|
for i in range(30):
|
||||||
|
screen.set_turn(f"u{i}", f"a{i}")
|
||||||
|
texts = _texts(screen)
|
||||||
|
assert len(texts) == 40
|
||||||
|
assert texts[0] == "a29"
|
||||||
|
assert "u0" not in texts and "u29" in texts
|
||||||
|
|
||||||
|
|
||||||
|
def test_scrolls_to_top_on_new_message(screen):
|
||||||
|
for i in range(12):
|
||||||
|
screen.set_turn(f"u{i}", "x " * 200)
|
||||||
|
QtTest.QTest.qWait(150) # lets the layout settle (and the first auto-scroll fire)
|
||||||
|
vbar = screen.transcript_scroll.verticalScrollBar()
|
||||||
|
assert vbar.maximum() > 0
|
||||||
|
vbar.setValue(vbar.maximum())
|
||||||
|
screen.set_turn("neu", "neu")
|
||||||
|
QtTest.QTest.qWait(150) # the scroll is deferred by a 50 ms timer
|
||||||
|
assert vbar.value() == vbar.minimum()
|
||||||
|
|
||||||
|
|
||||||
|
def test_fullscreen_button_goes_fullscreen_and_keeps_orb_and_chat_visible(qapp):
|
||||||
|
win = main_window.SlaveWindow()
|
||||||
|
win.show_main()
|
||||||
|
win.show()
|
||||||
|
scr = win.main_screen
|
||||||
|
scr.set_turn("frage", "antwort")
|
||||||
|
try:
|
||||||
|
assert not win.isFullScreen()
|
||||||
|
assert scr.expand_btn.toolTip() == "Vollbild"
|
||||||
|
|
||||||
|
scr.expand_btn.click()
|
||||||
|
QtTest.QTest.qWait(50)
|
||||||
|
assert win.isFullScreen()
|
||||||
|
assert scr.orb.isVisible() and scr.transcript_scroll.isVisible()
|
||||||
|
assert scr.expand_btn.toolTip() == "Vollbild beenden"
|
||||||
|
|
||||||
|
scr.expand_btn.click()
|
||||||
|
QtTest.QTest.qWait(50)
|
||||||
|
assert not win.isFullScreen()
|
||||||
|
assert scr.orb.isVisible() and scr.transcript_scroll.isVisible()
|
||||||
|
assert scr.expand_btn.toolTip() == "Vollbild"
|
||||||
|
finally:
|
||||||
|
win.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_esc_and_f11_keep_button_in_sync(qapp):
|
||||||
|
win = main_window.SlaveWindow()
|
||||||
|
win.show_main()
|
||||||
|
win.show()
|
||||||
|
try:
|
||||||
|
win._toggle_fullscreen() # F11
|
||||||
|
QtTest.QTest.qWait(50)
|
||||||
|
assert win.isFullScreen() and win.main_screen.expand_btn.toolTip() == "Vollbild beenden"
|
||||||
|
|
||||||
|
win._leave_fullscreen() # Esc
|
||||||
|
QtTest.QTest.qWait(50)
|
||||||
|
assert not win.isFullScreen() and win.main_screen.expand_btn.toolTip() == "Vollbild"
|
||||||
|
finally:
|
||||||
|
win.close()
|
||||||
|
|
||||||
|
def test_expand_icons_render(qapp):
|
||||||
|
for kind in ("expand", "collapse"):
|
||||||
|
img = main_window._icon(kind, "#cccccc", 26).pixmap(26, 26).toImage()
|
||||||
|
assert any(img.pixelColor(x, y).alpha() > 0
|
||||||
|
for x in range(img.width()) for y in range(img.height()))
|
||||||
@@ -0,0 +1,335 @@
|
|||||||
|
"""Unidentified-speaker inbox, sorting clips into profiles, and live samples."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from javis.core.config import Config, PathsConfig
|
||||||
|
from javis.core.events import EventBus
|
||||||
|
from javis.core.state import AgentState
|
||||||
|
from javis.stt.unidentified import UnidentifiedStore
|
||||||
|
from javis.stt.voice_profiler import NullProfiler, VoiceProfiler
|
||||||
|
|
||||||
|
SR = 16000
|
||||||
|
|
||||||
|
|
||||||
|
def _clip(voice: int, seconds: float = 1.0) -> np.ndarray:
|
||||||
|
"""Audio whose first sample encodes the (fake) speaker id."""
|
||||||
|
a = np.full(int(SR * seconds), 0.2, dtype=np.float32)
|
||||||
|
a[0] = voice / 100.0
|
||||||
|
return a
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_embed(audio: np.ndarray, sr: int) -> np.ndarray:
|
||||||
|
voice = round(float(audio[0]) * 100)
|
||||||
|
v = np.full(192, 0.01, dtype=np.float32)
|
||||||
|
v[voice] = 1.0
|
||||||
|
return v / np.linalg.norm(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _profiler(base: Path) -> VoiceProfiler:
|
||||||
|
prof = VoiceProfiler(Config(paths=PathsConfig(base_dir=base)))
|
||||||
|
prof._encoder = object() # pretend the ECAPA model is loaded
|
||||||
|
prof.embed = _fake_embed # type: ignore[method-assign]
|
||||||
|
return prof
|
||||||
|
|
||||||
|
|
||||||
|
class UnidentifiedStoreTests(unittest.TestCase):
|
||||||
|
def test_add_list_load_remove_and_persist(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
store = UnidentifiedStore(Path(d), max_clips=10)
|
||||||
|
emb = _fake_embed(_clip(3), SR)
|
||||||
|
a = store.add(_clip(3), SR, embedding=emb, text=" hallo ", source="local",
|
||||||
|
best_name="Bob", best_score=0.41)
|
||||||
|
b = store.add(_clip(4, 2.0), SR, text="zwei")
|
||||||
|
entries = store.entries()
|
||||||
|
self.assertEqual([e["id"] for e in entries], [b, a]) # newest first
|
||||||
|
self.assertEqual(entries[1]["text"], "hallo")
|
||||||
|
self.assertEqual(entries[1]["best_name"], "Bob")
|
||||||
|
self.assertAlmostEqual(entries[0]["duration_s"], 2.0)
|
||||||
|
|
||||||
|
audio, sr = store.load_audio(a)
|
||||||
|
self.assertEqual(sr, SR)
|
||||||
|
self.assertEqual(len(audio), SR)
|
||||||
|
np.testing.assert_allclose(store.load_embedding(a), emb, rtol=1e-6)
|
||||||
|
self.assertIsNone(store.load_embedding(b))
|
||||||
|
|
||||||
|
# A fresh instance reads the same index from disk.
|
||||||
|
self.assertEqual(len(UnidentifiedStore(Path(d))), 2)
|
||||||
|
|
||||||
|
self.assertEqual(store.remove([a, "nope"]), 1)
|
||||||
|
self.assertFalse((Path(d) / f"{a}.wav").exists())
|
||||||
|
self.assertFalse((Path(d) / f"{a}.npy").exists())
|
||||||
|
self.assertEqual(store.clear(), 1)
|
||||||
|
self.assertEqual(store.entries(), [])
|
||||||
|
with self.assertRaises(KeyError):
|
||||||
|
store.load_audio(a)
|
||||||
|
|
||||||
|
def test_oldest_clips_are_pruned(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
store = UnidentifiedStore(Path(d), max_clips=2)
|
||||||
|
ids = [store.add(_clip(i + 1), SR, text=str(i)) for i in range(3)]
|
||||||
|
self.assertEqual([e["id"] for e in store.entries()], [ids[2], ids[1]])
|
||||||
|
self.assertFalse((Path(d) / f"{ids[0]}.wav").exists())
|
||||||
|
|
||||||
|
|
||||||
|
class ProfilerEmbeddingTests(unittest.TestCase):
|
||||||
|
def test_enroll_rank_identify_refine(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
prof = _profiler(Path(d))
|
||||||
|
self.assertTrue(prof.enroll_embeddings("Alice", [_fake_embed(_clip(1), SR)] * 2))
|
||||||
|
prof.enroll("Bob", [_clip(2)], SR)
|
||||||
|
ranked = prof.rank(_fake_embed(_clip(1), SR))
|
||||||
|
self.assertEqual(ranked[0][0], "Alice")
|
||||||
|
self.assertGreater(ranked[0][1], 0.99)
|
||||||
|
self.assertEqual(prof.identify_embedding(_fake_embed(_clip(2), SR))[0], "Bob")
|
||||||
|
self.assertIsNone(prof.identify_embedding(_fake_embed(_clip(9), SR)))
|
||||||
|
self.assertEqual(prof.rank(np.zeros(192)), [])
|
||||||
|
|
||||||
|
# Running centroid: a Bob sample pulls Alice's centroid towards Bob.
|
||||||
|
before = dict(prof.rank(_fake_embed(_clip(2), SR)))["Alice"]
|
||||||
|
self.assertTrue(prof.refine_embedding("Alice", _fake_embed(_clip(2), SR)))
|
||||||
|
after = dict(prof.rank(_fake_embed(_clip(2), SR)))["Alice"]
|
||||||
|
self.assertGreater(after, before)
|
||||||
|
alice = next(p for p in prof.list_profiles() if p["name"] == "Alice")
|
||||||
|
self.assertEqual(alice["clip_count"], 3)
|
||||||
|
|
||||||
|
self.assertFalse(prof.refine_embedding("Nobody", _fake_embed(_clip(1), SR)))
|
||||||
|
self.assertFalse(prof.refine_embedding("Alice", np.ones(10))) # wrong shape
|
||||||
|
self.assertFalse(prof.enroll_embeddings("Zero", [np.zeros(192)]))
|
||||||
|
|
||||||
|
def test_sanitised_name_collision_gets_its_own_file(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
prof = _profiler(Path(d))
|
||||||
|
prof.enroll_embeddings("Anna B", [_fake_embed(_clip(1), SR)])
|
||||||
|
prof.enroll_embeddings("Anna_B", [_fake_embed(_clip(2), SR)])
|
||||||
|
files = {p["name"]: p["file"] for p in prof.list_profiles()}
|
||||||
|
self.assertNotEqual(files["Anna B"], files["Anna_B"])
|
||||||
|
prof2 = _profiler(Path(d))
|
||||||
|
prof2._load_index()
|
||||||
|
self.assertEqual(prof2.rank(_fake_embed(_clip(1), SR))[0][0], "Anna B")
|
||||||
|
|
||||||
|
def test_null_profiler_mirrors_api(self) -> None:
|
||||||
|
n = NullProfiler()
|
||||||
|
self.assertFalse(n.is_ready())
|
||||||
|
self.assertFalse(n.has_profile("x"))
|
||||||
|
self.assertEqual(n.rank(np.ones(192)), [])
|
||||||
|
self.assertFalse(n.enroll_embeddings("x", [np.ones(192)]))
|
||||||
|
self.assertFalse(n.refine_embedding("x", np.ones(192)))
|
||||||
|
self.assertIsNone(n.identify_embedding(np.ones(192)))
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeMic:
|
||||||
|
chunk_ms = 32
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.drains = 0
|
||||||
|
|
||||||
|
def drain(self) -> None:
|
||||||
|
self.drains += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _orchestrator(base: Path):
|
||||||
|
from javis.agent.orchestrator import Orchestrator
|
||||||
|
|
||||||
|
orch = object.__new__(Orchestrator)
|
||||||
|
orch.config = Config(paths=PathsConfig(base_dir=base))
|
||||||
|
orch.bus = EventBus()
|
||||||
|
orch.profiler = _profiler(base)
|
||||||
|
orch.unidentified = UnidentifiedStore(base / "voice_profiles" / "unidentified")
|
||||||
|
orch._enroll_clip_buffer = []
|
||||||
|
orch._enroll_ask_pending = False
|
||||||
|
orch._turn_lock = asyncio.Lock()
|
||||||
|
orch._mic_free = asyncio.Event()
|
||||||
|
orch._mic_free.set()
|
||||||
|
orch.mic = _FakeMic()
|
||||||
|
orch.vad = None
|
||||||
|
orch.state = AgentState.IDLE
|
||||||
|
events: list[tuple[str, object]] = []
|
||||||
|
|
||||||
|
async def _record(name, payload=None):
|
||||||
|
events.append((name, payload))
|
||||||
|
|
||||||
|
for ev in ("unidentified_changed", "voice_profiles_changed", "speaker_enrolled",
|
||||||
|
"speaker_ask_name"):
|
||||||
|
orch.bus.subscribe(ev, lambda p, _n=ev: _record(_n, p))
|
||||||
|
return orch, events
|
||||||
|
|
||||||
|
|
||||||
|
class AssignUnidentifiedTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_unknown_turn_is_stored_then_sorted_into_new_profile(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, events = _orchestrator(Path(d))
|
||||||
|
orch.config.speaker_id.auto_enroll = False
|
||||||
|
await orch._handle_unknown_speaker(_clip(5), SR, "wer bin ich", source="Küche")
|
||||||
|
await orch._handle_unknown_speaker(_clip(5), SR, " ", source="local") # no text
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
entries = orch.unidentified.entries()
|
||||||
|
self.assertEqual(len(entries), 1)
|
||||||
|
self.assertEqual(entries[0]["source"], "Küche")
|
||||||
|
self.assertIn(("unidentified_changed", {"added": entries[0]["id"]}), events)
|
||||||
|
|
||||||
|
res = await orch.assign_unidentified([entries[0]["id"]], " Carla ")
|
||||||
|
self.assertEqual(res, {"name": "Carla", "added": 1, "created": True})
|
||||||
|
self.assertTrue(orch.profiler.has_profile("Carla"))
|
||||||
|
self.assertEqual(orch.unidentified.entries(), [])
|
||||||
|
self.assertEqual(orch.profiler.identify_embedding(_fake_embed(_clip(5), SR))[0],
|
||||||
|
"Carla")
|
||||||
|
names = [e[0] for e in events]
|
||||||
|
self.assertIn("voice_profiles_changed", names)
|
||||||
|
|
||||||
|
async def test_assign_to_existing_profile_refines_it(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, _ = _orchestrator(Path(d))
|
||||||
|
orch.profiler.enroll("Dora", [_clip(6)], SR)
|
||||||
|
ids = [orch.unidentified.add(_clip(6), SR, text="a"),
|
||||||
|
orch.unidentified.add(_clip(6), SR, text="b")] # no stored embedding
|
||||||
|
res = await orch.assign_unidentified(ids, "Dora")
|
||||||
|
self.assertEqual(res["added"], 2)
|
||||||
|
self.assertFalse(res["created"])
|
||||||
|
dora = next(p for p in orch.profiler.list_profiles() if p["name"] == "Dora")
|
||||||
|
self.assertEqual(dora["clip_count"], 3)
|
||||||
|
|
||||||
|
async def test_assign_validates_input(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, _ = _orchestrator(Path(d))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
await orch.assign_unidentified(["x"], " ")
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
await orch.assign_unidentified(["missing"], "Eve")
|
||||||
|
orch.profiler._encoder = None
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
await orch.assign_unidentified(["x"], "Eve")
|
||||||
|
|
||||||
|
async def test_name_prompt_confirm_removes_inbox_copies(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, events = _orchestrator(Path(d))
|
||||||
|
await orch._handle_unknown_speaker(_clip(7), SR, "hallo", source="local")
|
||||||
|
await orch._handle_unknown_speaker(_clip(7), SR, "noch mal", source="local")
|
||||||
|
self.assertEqual(len(orch.pending_enroll_clips()), 2)
|
||||||
|
self.assertEqual(sum(1 for e in events if e[0] == "speaker_ask_name"), 1)
|
||||||
|
self.assertTrue(await orch.auto_enroll_confirm("k", "Frank"))
|
||||||
|
self.assertEqual(orch.unidentified.entries(), [])
|
||||||
|
self.assertTrue(orch.profiler.has_profile("Frank"))
|
||||||
|
self.assertIn(("speaker_enrolled", {"name": "Frank", "clip_count": 2}), events)
|
||||||
|
|
||||||
|
async def test_dismiss_keeps_inbox_and_allows_new_prompt(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, events = _orchestrator(Path(d))
|
||||||
|
await orch._handle_unknown_speaker(_clip(8), SR, "hallo", source="local")
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
orch.auto_enroll_dismiss()
|
||||||
|
self.assertEqual(orch.pending_enroll_clips(), [])
|
||||||
|
self.assertEqual(len(orch.unidentified.entries()), 1)
|
||||||
|
await orch._handle_unknown_speaker(_clip(8), SR, "wieder", source="local")
|
||||||
|
self.assertEqual(sum(1 for e in events if e[0] == "speaker_ask_name"), 2)
|
||||||
|
|
||||||
|
|
||||||
|
class LiveSampleTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_record_and_add_samples(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, events = _orchestrator(Path(d))
|
||||||
|
orch.profiler.enroll("Gina", [_clip(9)], SR)
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
async def fake_record(config, mic, vad, **kw):
|
||||||
|
# The wake loop must be paused and the turn lock held meanwhile.
|
||||||
|
seen["mic_free"] = orch._mic_free.is_set()
|
||||||
|
seen["locked"] = orch._turn_lock.locked()
|
||||||
|
seen["state"] = orch.state
|
||||||
|
return _clip(9)
|
||||||
|
|
||||||
|
async def no_chime():
|
||||||
|
return None
|
||||||
|
|
||||||
|
orch._play_wake_chime = no_chime
|
||||||
|
with patch("javis.agent.orchestrator.record_utterance", fake_record):
|
||||||
|
sample = await orch.record_voice_sample("Gina")
|
||||||
|
self.assertEqual(seen, {"mic_free": False, "locked": True,
|
||||||
|
"state": AgentState.ENROLLING})
|
||||||
|
self.assertTrue(orch._mic_free.is_set())
|
||||||
|
self.assertFalse(orch._turn_lock.locked())
|
||||||
|
self.assertEqual(orch.state, AgentState.IDLE)
|
||||||
|
self.assertGreater(sample.score, 0.99)
|
||||||
|
self.assertEqual(sample.best_name, "Gina")
|
||||||
|
self.assertAlmostEqual(sample.duration_s, 1.0)
|
||||||
|
|
||||||
|
used = await orch.add_voice_samples("Gina", [sample, sample])
|
||||||
|
self.assertEqual(used, 2)
|
||||||
|
gina = next(p for p in orch.profiler.list_profiles() if p["name"] == "Gina")
|
||||||
|
self.assertEqual(gina["clip_count"], 3)
|
||||||
|
self.assertIn(("voice_profiles_changed",
|
||||||
|
{"name": "Gina", "added": 2, "created": False}), events)
|
||||||
|
|
||||||
|
async def test_record_errors_are_user_facing(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, _ = _orchestrator(Path(d))
|
||||||
|
|
||||||
|
async def no_speech(*a, **kw):
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def no_chime():
|
||||||
|
return None
|
||||||
|
|
||||||
|
orch._play_wake_chime = no_chime
|
||||||
|
with patch("javis.agent.orchestrator.record_utterance", no_speech), \
|
||||||
|
self.assertRaisesRegex(RuntimeError, "No speech"):
|
||||||
|
await orch.record_voice_sample("Gina")
|
||||||
|
self.assertTrue(orch._mic_free.is_set())
|
||||||
|
self.assertFalse(orch._turn_lock.locked())
|
||||||
|
|
||||||
|
async with orch._turn_lock: # a conversation turn is running
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "busy"):
|
||||||
|
async with orch._exclusive_mic(wait_s=0.05):
|
||||||
|
pass
|
||||||
|
self.assertTrue(orch._mic_free.is_set())
|
||||||
|
|
||||||
|
orch.mic = None
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "microphone"):
|
||||||
|
await orch.record_voice_sample("Gina")
|
||||||
|
|
||||||
|
async def test_wake_loop_yields_mic_to_exclusive_recording(self) -> None:
|
||||||
|
from javis.agent.orchestrator import Orchestrator
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
orch, _ = _orchestrator(Path(d))
|
||||||
|
orch._stop = asyncio.Event()
|
||||||
|
fed: list[int] = []
|
||||||
|
|
||||||
|
class Wake:
|
||||||
|
def reset(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def feed(self, chunk):
|
||||||
|
fed.append(len(chunk))
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
def triggered(self, score):
|
||||||
|
return False
|
||||||
|
|
||||||
|
class Mic(_FakeMic):
|
||||||
|
async def chunks(self, stale_timeout=None):
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(0.005)
|
||||||
|
yield np.zeros(512, dtype=np.float32)
|
||||||
|
|
||||||
|
orch.wake = Wake()
|
||||||
|
orch.mic = Mic()
|
||||||
|
orch._mic_free.clear()
|
||||||
|
task = asyncio.ensure_future(Orchestrator._wait_for_wake(orch))
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
self.assertEqual(fed, []) # waits while the mic is taken
|
||||||
|
orch._mic_free.set()
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
self.assertTrue(fed) # resumed wake detection
|
||||||
|
orch._mic_free.clear()
|
||||||
|
self.assertFalse(await asyncio.wait_for(task, 1.0))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+2705
-4
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user