From 435c645f36971dfca2144f4b212991c7cf097ab2 Mon Sep 17 00:00:00 2001 From: Javis Date: Mon, 21 Sep 2026 21:44:00 +0200 Subject: [PATCH] Initial commit: Javis project --- .clinerules/Windows-env.md | 127 + .clinerules/workflows/STRICT.md | 12 + .gitignore | 68 + Javis.spec | 73 + README.md | 131 + config.example.yaml | 117 + docs/WAKE_WORD.md | 130 + duckduckgo-mcp/README.md | 26 + duckduckgo-mcp/pyproject.toml | 23 + duckduckgo-mcp/run.bat | 15 + duckduckgo-mcp/src/duckduckgo_mcp/__init__.py | 1 + duckduckgo-mcp/src/duckduckgo_mcp/__main__.py | 5 + duckduckgo-mcp/src/duckduckgo_mcp/server.py | 199 + emby-mcp/.env.example | 12 + emby-mcp/README.md | 138 + emby-mcp/mcp-emby.example.json | 14 + emby-mcp/pyproject.toml | 33 + emby-mcp/run-lan.bat | 20 + emby-mcp/run.bat | 15 + emby-mcp/src/emby_mcp/__init__.py | 3 + emby-mcp/src/emby_mcp/__main__.py | 4 + emby-mcp/src/emby_mcp/client.py | 134 + emby-mcp/src/emby_mcp/config.py | 40 + emby-mcp/src/emby_mcp/server.py | 438 ++ homey-mcp/.env.example | 16 + homey-mcp/.gitignore | 22 + homey-mcp/README.md | 200 + homey-mcp/UBUNTU_SETUP.md | 137 + homey-mcp/homey-mcp.service | 27 + homey-mcp/mcp-odysseus.example.json | 11 + homey-mcp/pyproject.toml | 40 + homey-mcp/run-lan.bat | 22 + homey-mcp/run-lan.sh | 34 + homey-mcp/run.bat | 15 + homey-mcp/run.sh | 22 + homey-mcp/setup-ubuntu.sh | 85 + homey-mcp/src/homey_mcp/__init__.py | 3 + homey-mcp/src/homey_mcp/__main__.py | 4 + homey-mcp/src/homey_mcp/config.py | 42 + homey-mcp/src/homey_mcp/homey_client.py | 180 + homey-mcp/src/homey_mcp/server.py | 1383 +++++ javis-slave-mcp/README.md | 150 + javis-slave-mcp/pyproject.toml | 26 + javis-slave-mcp/run.bat | 15 + .../src/javis_slave_mcp/__init__.py | 2 + .../src/javis_slave_mcp/__main__.py | 21 + javis-slave-mcp/src/javis_slave_mcp/server.py | 945 +++ javis-slave/RASPBERRY_PI.md | 162 + javis-slave/README.md | 89 + javis-slave/javis-slave.service | 25 + javis-slave/pyproject.toml | 52 + javis-slave/run.bat | 16 + javis-slave/run.sh | 35 + javis-slave/setup-pi.sh | 122 + javis-slave/src/javis_slave/__init__.py | 2 + javis-slave/src/javis_slave/__main__.py | 128 + javis-slave/src/javis_slave/api_client.py | 286 + javis-slave/src/javis_slave/app.py | 282 + javis-slave/src/javis_slave/audio_engine.py | 395 ++ .../src/javis_slave/audio_io/__init__.py | 6 + .../src/javis_slave/audio_io/capture.py | 83 + .../src/javis_slave/audio_io/stream.py | 404 ++ javis-slave/src/javis_slave/audio_io/vad.py | 96 + .../src/javis_slave/audio_io/wake_word.py | 109 + javis-slave/src/javis_slave/config.py | 90 + javis-slave/src/javis_slave/headless.py | 183 + javis-slave/src/javis_slave/http_server.py | 444 ++ javis-slave/src/javis_slave/logging_setup.py | 46 + javis-slave/src/javis_slave/main_window.py | 584 ++ javis-slave/src/javis_slave/system_control.py | 558 ++ .../src/javis_slave/windows_control.py | 558 ++ mcp_servers.example.json | 24 + pyproject.toml | 50 + requirements.txt | 69 + run.bat | 109 + spotify-mcp-server/.github/dependabot.yml | 10 + spotify-mcp-server/.github/workflows/ci.yml | 37 + spotify-mcp-server/.gitignore | 8 + spotify-mcp-server/README.md | 411 ++ spotify-mcp-server/biome.jsonc | 149 + spotify-mcp-server/package.json | 31 + .../spotify-config.example.json | 8 + spotify-mcp-server/src/albums.ts | 304 + spotify-mcp-server/src/auth.ts | 14 + spotify-mcp-server/src/index.ts | 27 + spotify-mcp-server/src/play.ts | 586 ++ spotify-mcp-server/src/playlist.ts | 294 + spotify-mcp-server/src/read.ts | 766 +++ spotify-mcp-server/src/types.ts | 53 + spotify-mcp-server/src/utils.ts | 439 ++ spotify-mcp-server/tsconfig.json | 15 + spotify-mcp-server/verify_tools.mjs | 171 + src/javis/__init__.py | 2 + src/javis/__main__.py | 98 + src/javis/agent/__init__.py | 0 src/javis/agent/middleware.py | 1842 ++++++ src/javis/agent/orchestrator.py | 3300 ++++++++++ src/javis/agent/prompts.py | 408 ++ src/javis/agent/quick_engine.py | 230 + src/javis/api/__init__.py | 1 + src/javis/api/devices.py | 159 + src/javis/api/models.py | 52 + src/javis/api/pairing.py | 42 + src/javis/api/routes.py | 205 + src/javis/api/server.py | 58 + src/javis/audio_io/__init__.py | 0 src/javis/audio_io/capture.py | 128 + src/javis/audio_io/devices.py | 45 + src/javis/audio_io/stream.py | 224 + src/javis/audio_io/vad.py | 101 + src/javis/audio_io/wake_word.py | 113 + src/javis/core/__init__.py | 0 src/javis/core/bootstrap.py | 174 + src/javis/core/config.py | 536 ++ src/javis/core/config_io.py | 73 + src/javis/core/doctor.py | 103 + src/javis/core/events.py | 50 + src/javis/core/hardware.py | 309 + src/javis/core/logger.py | 68 + src/javis/core/state.py | 14 + src/javis/core/timer.py | 194 + src/javis/gui/__init__.py | 0 src/javis/gui/hotkey.py | 100 + src/javis/gui/main_window.py | 5341 +++++++++++++++++ src/javis/gui/startup_dialog.py | 1043 ++++ src/javis/gui/theme.py | 42 + src/javis/llm/__init__.py | 0 src/javis/llm/conversation.py | 139 + src/javis/llm/hermes_client.py | 147 + src/javis/llm/ollama_client.py | 383 ++ src/javis/mcp_client/__init__.py | 0 src/javis/mcp_client/host.py | 463 ++ src/javis/mcp_client/schema_bridge.py | 65 + src/javis/stt/__init__.py | 0 src/javis/stt/transcriber.py | 252 + src/javis/stt/voice_profiler.py | 413 ++ src/javis/tts/__init__.py | 0 src/javis/tts/base.py | 32 + src/javis/tts/f5_backend.py | 152 + src/javis/tts/factory.py | 19 + src/javis/tts/models.py | 94 + src/javis/tts/piper_backend.py | 160 + src/javis/tts/runner.py | 26 + src/javis/voices/__init__.py | 0 src/javis/voices/audiobook_ingest.py | 794 +++ src/javis/voices/finetune.py | 95 + src/javis/voices/io.py | 269 + src/javis/voices/manager.py | 85 + src/javis/voices/piper_prep.py | 178 + src/javis/voices/piper_train.py | 529 ++ src/javis/voices/profile.py | 75 + src/javis/voices/training_scheduler.py | 310 + tests/_retheme_check.py | 118 + tests/test_fixes.py | 404 ++ tests/test_hermes_memory.py | 120 + voice-ref-curator/VoiceRefCurator.spec | 44 + voice-ref-curator/build.bat | 24 + voice-ref-curator/pyproject.toml | 22 + voice-ref-curator/run.bat | 6 + .../src/voice_ref_curator/__init__.py | 1 + .../src/voice_ref_curator/__main__.py | 10 + .../src/voice_ref_curator/analyser.py | 349 ++ .../src/voice_ref_curator/gui.py | 708 +++ .../src/voice_ref_curator/speaker.py | 228 + 164 files changed, 35279 insertions(+) create mode 100644 .clinerules/Windows-env.md create mode 100644 .clinerules/workflows/STRICT.md create mode 100644 .gitignore create mode 100644 Javis.spec create mode 100644 README.md create mode 100644 config.example.yaml create mode 100644 docs/WAKE_WORD.md create mode 100644 duckduckgo-mcp/README.md create mode 100644 duckduckgo-mcp/pyproject.toml create mode 100644 duckduckgo-mcp/run.bat create mode 100644 duckduckgo-mcp/src/duckduckgo_mcp/__init__.py create mode 100644 duckduckgo-mcp/src/duckduckgo_mcp/__main__.py create mode 100644 duckduckgo-mcp/src/duckduckgo_mcp/server.py create mode 100644 emby-mcp/.env.example create mode 100644 emby-mcp/README.md create mode 100644 emby-mcp/mcp-emby.example.json create mode 100644 emby-mcp/pyproject.toml create mode 100644 emby-mcp/run-lan.bat create mode 100644 emby-mcp/run.bat create mode 100644 emby-mcp/src/emby_mcp/__init__.py create mode 100644 emby-mcp/src/emby_mcp/__main__.py create mode 100644 emby-mcp/src/emby_mcp/client.py create mode 100644 emby-mcp/src/emby_mcp/config.py create mode 100644 emby-mcp/src/emby_mcp/server.py create mode 100644 homey-mcp/.env.example create mode 100644 homey-mcp/.gitignore create mode 100644 homey-mcp/README.md create mode 100644 homey-mcp/UBUNTU_SETUP.md create mode 100644 homey-mcp/homey-mcp.service create mode 100644 homey-mcp/mcp-odysseus.example.json create mode 100644 homey-mcp/pyproject.toml create mode 100644 homey-mcp/run-lan.bat create mode 100644 homey-mcp/run-lan.sh create mode 100644 homey-mcp/run.bat create mode 100644 homey-mcp/run.sh create mode 100644 homey-mcp/setup-ubuntu.sh create mode 100644 homey-mcp/src/homey_mcp/__init__.py create mode 100644 homey-mcp/src/homey_mcp/__main__.py create mode 100644 homey-mcp/src/homey_mcp/config.py create mode 100644 homey-mcp/src/homey_mcp/homey_client.py create mode 100644 homey-mcp/src/homey_mcp/server.py create mode 100644 javis-slave-mcp/README.md create mode 100644 javis-slave-mcp/pyproject.toml create mode 100644 javis-slave-mcp/run.bat create mode 100644 javis-slave-mcp/src/javis_slave_mcp/__init__.py create mode 100644 javis-slave-mcp/src/javis_slave_mcp/__main__.py create mode 100644 javis-slave-mcp/src/javis_slave_mcp/server.py create mode 100644 javis-slave/RASPBERRY_PI.md create mode 100644 javis-slave/README.md create mode 100644 javis-slave/javis-slave.service create mode 100644 javis-slave/pyproject.toml create mode 100644 javis-slave/run.bat create mode 100755 javis-slave/run.sh create mode 100644 javis-slave/setup-pi.sh create mode 100644 javis-slave/src/javis_slave/__init__.py create mode 100644 javis-slave/src/javis_slave/__main__.py create mode 100644 javis-slave/src/javis_slave/api_client.py create mode 100644 javis-slave/src/javis_slave/app.py create mode 100644 javis-slave/src/javis_slave/audio_engine.py create mode 100644 javis-slave/src/javis_slave/audio_io/__init__.py create mode 100644 javis-slave/src/javis_slave/audio_io/capture.py create mode 100644 javis-slave/src/javis_slave/audio_io/stream.py create mode 100644 javis-slave/src/javis_slave/audio_io/vad.py create mode 100644 javis-slave/src/javis_slave/audio_io/wake_word.py create mode 100644 javis-slave/src/javis_slave/config.py create mode 100644 javis-slave/src/javis_slave/headless.py create mode 100644 javis-slave/src/javis_slave/http_server.py create mode 100644 javis-slave/src/javis_slave/logging_setup.py create mode 100644 javis-slave/src/javis_slave/main_window.py create mode 100644 javis-slave/src/javis_slave/system_control.py create mode 100644 javis-slave/src/javis_slave/windows_control.py create mode 100644 mcp_servers.example.json create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 run.bat create mode 100644 spotify-mcp-server/.github/dependabot.yml create mode 100644 spotify-mcp-server/.github/workflows/ci.yml create mode 100644 spotify-mcp-server/.gitignore create mode 100644 spotify-mcp-server/README.md create mode 100644 spotify-mcp-server/biome.jsonc create mode 100644 spotify-mcp-server/package.json create mode 100644 spotify-mcp-server/spotify-config.example.json create mode 100644 spotify-mcp-server/src/albums.ts create mode 100644 spotify-mcp-server/src/auth.ts create mode 100644 spotify-mcp-server/src/index.ts create mode 100644 spotify-mcp-server/src/play.ts create mode 100644 spotify-mcp-server/src/playlist.ts create mode 100644 spotify-mcp-server/src/read.ts create mode 100644 spotify-mcp-server/src/types.ts create mode 100644 spotify-mcp-server/src/utils.ts create mode 100644 spotify-mcp-server/tsconfig.json create mode 100644 spotify-mcp-server/verify_tools.mjs create mode 100644 src/javis/__init__.py create mode 100644 src/javis/__main__.py create mode 100644 src/javis/agent/__init__.py create mode 100644 src/javis/agent/middleware.py create mode 100644 src/javis/agent/orchestrator.py create mode 100644 src/javis/agent/prompts.py create mode 100644 src/javis/agent/quick_engine.py create mode 100644 src/javis/api/__init__.py create mode 100644 src/javis/api/devices.py create mode 100644 src/javis/api/models.py create mode 100644 src/javis/api/pairing.py create mode 100644 src/javis/api/routes.py create mode 100644 src/javis/api/server.py create mode 100644 src/javis/audio_io/__init__.py create mode 100644 src/javis/audio_io/capture.py create mode 100644 src/javis/audio_io/devices.py create mode 100644 src/javis/audio_io/stream.py create mode 100644 src/javis/audio_io/vad.py create mode 100644 src/javis/audio_io/wake_word.py create mode 100644 src/javis/core/__init__.py create mode 100644 src/javis/core/bootstrap.py create mode 100644 src/javis/core/config.py create mode 100644 src/javis/core/config_io.py create mode 100644 src/javis/core/doctor.py create mode 100644 src/javis/core/events.py create mode 100644 src/javis/core/hardware.py create mode 100644 src/javis/core/logger.py create mode 100644 src/javis/core/state.py create mode 100644 src/javis/core/timer.py create mode 100644 src/javis/gui/__init__.py create mode 100644 src/javis/gui/hotkey.py create mode 100644 src/javis/gui/main_window.py create mode 100644 src/javis/gui/startup_dialog.py create mode 100644 src/javis/gui/theme.py create mode 100644 src/javis/llm/__init__.py create mode 100644 src/javis/llm/conversation.py create mode 100644 src/javis/llm/hermes_client.py create mode 100644 src/javis/llm/ollama_client.py create mode 100644 src/javis/mcp_client/__init__.py create mode 100644 src/javis/mcp_client/host.py create mode 100644 src/javis/mcp_client/schema_bridge.py create mode 100644 src/javis/stt/__init__.py create mode 100644 src/javis/stt/transcriber.py create mode 100644 src/javis/stt/voice_profiler.py create mode 100644 src/javis/tts/__init__.py create mode 100644 src/javis/tts/base.py create mode 100644 src/javis/tts/f5_backend.py create mode 100644 src/javis/tts/factory.py create mode 100644 src/javis/tts/models.py create mode 100644 src/javis/tts/piper_backend.py create mode 100644 src/javis/tts/runner.py create mode 100644 src/javis/voices/__init__.py create mode 100644 src/javis/voices/audiobook_ingest.py create mode 100644 src/javis/voices/finetune.py create mode 100644 src/javis/voices/io.py create mode 100644 src/javis/voices/manager.py create mode 100644 src/javis/voices/piper_prep.py create mode 100644 src/javis/voices/piper_train.py create mode 100644 src/javis/voices/profile.py create mode 100644 src/javis/voices/training_scheduler.py create mode 100644 tests/_retheme_check.py create mode 100644 tests/test_fixes.py create mode 100644 tests/test_hermes_memory.py create mode 100644 voice-ref-curator/VoiceRefCurator.spec create mode 100644 voice-ref-curator/build.bat create mode 100644 voice-ref-curator/pyproject.toml create mode 100644 voice-ref-curator/run.bat create mode 100644 voice-ref-curator/src/voice_ref_curator/__init__.py create mode 100644 voice-ref-curator/src/voice_ref_curator/__main__.py create mode 100644 voice-ref-curator/src/voice_ref_curator/analyser.py create mode 100644 voice-ref-curator/src/voice_ref_curator/gui.py create mode 100644 voice-ref-curator/src/voice_ref_curator/speaker.py diff --git a/.clinerules/Windows-env.md b/.clinerules/Windows-env.md new file mode 100644 index 0000000..98af7e6 --- /dev/null +++ b/.clinerules/Windows-env.md @@ -0,0 +1,127 @@ +# Windows Environment Rules + +## Execution Environment +- You are running on Microsoft Windows. +- The default shell is PowerShell (pwsh or Windows PowerShell), NOT bash. +- Generate PowerShell commands unless explicitly asked otherwise. +- Never use bash syntax, zsh syntax, or Linux-only commands. +- Never assume WSL, Git Bash, Cygwin, or MSYS2 are available. +- Never assume a Unix environment. + +## Command Syntax +- Use PowerShell syntax. +- Use backslashes (\) for Windows paths unless a tool explicitly requires forward slashes. +- Quote paths that may contain spaces. +- Prefer: + Get-ChildItem + Get-Content + Set-Content + Copy-Item + Move-Item + Remove-Item + New-Item + Test-Path + Resolve-Path +instead of: + ls + cat + cp + mv + rm + touch + +- Use: + Get-ChildItem -Force +instead of: + ls -la + +- Use: + Select-String +instead of: + grep + +- Use: + Get-Process +instead of: + ps + +- Use: + Get-Command +instead of: + which + +- Use: + Get-Location +instead of: + pwd + +## PowerShell Best Practices +- Prefer PowerShell cmdlets over legacy CMD commands. +- Do not use cmd.exe syntax unless specifically requested. +- Do not use && or || if compatibility with Windows PowerShell 5.1 matters. Instead use proper PowerShell conditionals. +- Assume UTF-8 when writing files if possible. +- Use `$env:VARIABLE` for environment variables. +- Do not use `$VARIABLE` for environment variables. +- Exit codes are available in `$LASTEXITCODE`. + +## File System +- Windows paths are case-insensitive. +- Drives may be C:, D:, etc. +- Do not assume `/tmp`; use `$env:TEMP`. +- Do not assume `/home`; use `$HOME`. +- Do not assume `/usr/bin`. +- Do not assume `/etc`. +- Do not use `/dev/null`; use `$null` or `Out-Null` where appropriate. + +## Development Tools +Assume these may exist, but verify before relying on them: +- git +- node +- npm +- pnpm +- python +- dotnet +- cargo +- go + +When using a tool, first verify it exists if required. + +## Package Managers +Windows package managers may include: +- winget +- chocolatey (choco) +- scoop + +Do not suggest apt, yum, dnf, pacman, apk, or brew unless explicitly requested. + +## VS Code +- Assume work is being performed inside Visual Studio Code. +- Paths should be compatible with Windows. +- Use relative paths when possible. + +## Scripts +When writing scripts: +- Default to PowerShell (.ps1). +- Only write Bash scripts when explicitly requested. + +## Safety +- Never assume administrator privileges. +- Mention when a command requires elevation. +- Avoid destructive commands unless explicitly requested. +- Explain commands before suggesting system-wide changes. + +## Platform Detection +Unless the user explicitly states otherwise, always assume: +- Operating System: Windows 10/11 +- Shell: PowerShell +- Terminal encoding: UTF-8 +- Filesystem: NTFS +- Path separator: '\' + +## General Behavior +- Do not provide Linux commands alongside Windows commands unless the user requests cross-platform instructions. +- If documentation commonly uses bash, translate examples into PowerShell. +- When installing software, prefer `winget` if available. +- Before suggesting platform-specific fixes, ensure they apply to Windows. +- Never assume POSIX utilities are installed. +- Prefer native Windows solutions over third-party Unix compatibility layers. \ No newline at end of file diff --git a/.clinerules/workflows/STRICT.md b/.clinerules/workflows/STRICT.md new file mode 100644 index 0000000..ac4845d --- /dev/null +++ b/.clinerules/workflows/STRICT.md @@ -0,0 +1,12 @@ +## Workflow Rules + +- Think before acting. +- Read existing files before modifying them. +- Make the smallest possible change. +- Do not rewrite unrelated code. +- Preserve existing formatting and style. +- If a command may fail because a tool is missing, check for the tool first. +- Never invent file contents—read files before referencing them. +- Ask for clarification if requirements are ambiguous instead of making assumptions. +- Prefer robust, production-ready solutions over quick hacks. +- When generating commands, ensure they can be copied directly into PowerShell without modification. \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b5be43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +[python.gitignore] +# Python cache +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg-info/ +*.egg/ +dist/ +build/ +*.spec.bak + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ +.python-version + +# IDE / Editor +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# OS artifacts +Thumbs.db +desktop.ini + +# Logs +*.log +err.log +out.log + +# Config files with secrets +config.yaml +mcp_servers.json +.env + +# PyInstaller artifacts +*.spec.bak +build/ +dist/ + +# Node / npm (spotify-mcp-server) +node_modules/ +package-lock.json +npm-debug.log + +# TypeScript +*.tsbuildinfo + +# Continuing AI assistant +.continue/ + +# Temporary files +*.tmp +*.bak + +# Downloaded ML models (large binaries, re-download on first run) +.javis/models/ +models/ +*.bin +*.ckpt +*.pt +*.pth diff --git a/Javis.spec b/Javis.spec new file mode 100644 index 0000000..3239b7e --- /dev/null +++ b/Javis.spec @@ -0,0 +1,73 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_data_files +from PyInstaller.utils.hooks import collect_all + +datas = [('config.yaml', '.'), ('mcp_servers.json', '.')] +binaries = [] +hiddenimports = [] +datas += collect_data_files('javis') +tmp_ret = collect_all('PySide6') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('shiboken6') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('qasync') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('piper') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +# Optional: F5-TTS voice-cloning backend (uncomment if using tts.backend: f5) +# tmp_ret = collect_all('f5_tts') +# datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('faster_whisper') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('silero_vad') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('openwakeword') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('sounddevice') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('soundfile') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + + +a = Analysis( + ['src\\javis\\__main__.py'], + pathex=['src'], + binaries=binaries, + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='Javis', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, + icon='NONE', +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='Javis', +) diff --git a/README.md b/README.md new file mode 100644 index 0000000..08dbe35 --- /dev/null +++ b/README.md @@ -0,0 +1,131 @@ +# Javis — Local Voice AI Agent + +Wake-word activated, Ollama-backed, MCP-enabled voice assistant with audiobook-driven voice cloning. + +## Architecture + +``` +microphone ─▶ VAD / wake word ─▶ STT (faster-whisper) + │ + ▼ + Ollama LLM (llama3.1:8b) ◀─▶ MCP servers (filesystem, fetch, time, …) + │ + ▼ + TTS (XTTS v2, cloned voice) + │ + ▼ + speaker +``` + +- **STT** — faster-whisper, CUDA float16, default `large-v3`. +- **TTS** — Coqui XTTS v2 (community fork). Zero-shot cloning plus optional fine-tune from audiobook snippets. +- **LLM** — local Ollama at `http://localhost:11434`, `llama3.1:8b` (native tool calling). +- **MCP** — host/client only, reads `mcp_servers.json` (VS Code-compatible). +- **Wake word** — openWakeWord, default `hey_jarvis` (custom "Javis" model trainable later). +- **GUI** — PySide6 + qasync. + +## Prerequisites + +- Windows 10/11 +- Python 3.10 or 3.11 (XTTS does not yet support 3.12) +- NVIDIA GPU + recent CUDA drivers (CPU works but slow) +- A running Ollama instance with the chosen model pulled: + ```powershell + ollama pull llama3.1:8b + ``` +- For MCP servers: Node.js (for `npx`) and/or `uv` (`uvx`). + +## Install + +```powershell +cd Javis +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -e . # all Python deps (a CPU torch may be pulled in as a fallback) + +# Detects your GPU / driver and swaps in the matching PyTorch wheel (skip on CPU-only) +python -m javis --install-deps + +copy config.example.yaml config.yaml +copy mcp_servers.example.json mcp_servers.json + +# Verify: Ollama reachable, model pulled, CUDA/torch loadable, audio libs present +python -m javis --doctor +``` + +
+Manual install (if you prefer to pick the torch wheel yourself) + +```powershell +# CUDA torch first (skip if CPU-only) — pick the index matching your driver +pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu124 +pip install -r requirements.txt +pip install -e . +``` + +If torch later fails to import (DLL errors), run `python -m javis --repair`. +
+ +## Run + +```powershell +.\run.bat # GUI +python -m javis # GUI +python -m javis --headless # no GUI (tool calls with approval "ask" are denied — set "allow" per server) +python -m javis --doctor # check environment +``` + +First launch opens a configuration dialog (models, microphone with live level meter, +persona). Untick *Show this dialog on every start* to skip it next time; reopen it via +**Tools → Startup configuration…**. Closing the window hides Javis to the system tray so +wake word and timers keep running — quit via **File → Quit** or the tray menu. +Push-to-talk: `Ctrl+Space` in the window, `Ctrl+Alt+Space` system-wide (Windows). + +## Voice profiles + +1. Open the **Voices** panel. +2. Drag an audiobook file (mp3/wav/flac) in. +3. Javis will VAD-split, transcribe, and build a clean dataset. +4. Test with zero-shot cloning (instant) or click **Fine-tune** for higher quality. + +## MCP servers + +Edit `mcp_servers.json`. Schema matches VS Code's `mcp.json`, so you can copy/paste server entries. +First-time tool calls require approval (configurable per server). + +## Custom wake word + +The default wake phrase is *hey Jarvis*. To use your own name (e.g. **Rufus**), train a +small openWakeWord model on synthetic speech (< 1 h, Colab or WSL2), drop `rufus.onnx` +into `~/.javis/models/wakeword/` and set `wake_word.model: rufus`. Slaves fetch the model +from the master automatically. Step-by-step: [docs/WAKE_WORD.md](docs/WAKE_WORD.md). + +## Satellites (multi-room) + +[javis-slave](javis-slave/README.md) turns a Windows tablet or a **Raspberry Pi** into a +satellite: local wake word, audio streamed to this master over the LAN, spoken reply +played back locally. Enable `api.enabled: true` here, generate a pairing code in the +**Devices** tab, then follow [javis-slave/RASPBERRY_PI.md](javis-slave/RASPBERRY_PI.md) +for a headless Pi (`sudo ./setup-pi.sh`) or run `run.bat` on a tablet. +[javis-slave-mcp](javis-slave-mcp/README.md) exposes the satellites' volume/mute/power +controls as tools so you can say "mach die Küche leiser". + +## Project layout + +``` +Javis/ + src/javis/ + core/ config, logger, state machine, event bus + audio_io/ devices, mic stream, VAD, wake word + stt/ faster-whisper + tts/ backend interface, XTTS implementation + voices/ profiles, audiobook ingest, fine-tune + llm/ Ollama client, conversation memory + mcp_client/ host, schema bridge + agent/ orchestrator, system prompts + gui/ PySide6 main window + panels +``` + +## Status + +Initial scaffold. End-to-end voice loop works once dependencies and a voice reference clip are provided. diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..8fe02a2 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,117 @@ +# Javis configuration. Copy to config.yaml and edit. + +gui: + theme: "dark" # dark | light (Tools → "Toggle light/dark theme") + show_startup_dialog: true # show the configuration dialog before the main window + minimize_to_tray: true # closing the window hides to the tray; quit via File → Quit + global_ptt_hotkey: "ctrl+alt+space" # system-wide push-to-talk (Windows); "" = off + +llm: + base_url: "http://localhost:11434" + model: "gemma3:4b" + temperature: 0.7 + request_timeout_seconds: 120 + # Keep the model loaded in Ollama for 1 h after each request (default is + # only 5 min, so gaps > 5 min force a full model reload next turn). + # "infinity" keeps it loaded forever. + keep_alive: 3600 + +stt: + model_size: "large-v3" # tiny, base, small, medium, large-v3, distil-large-v3 + device: "auto" # auto, cuda, cpu + compute_type: "float16" # float16, int8_float16, int8, float32 + language: null # null = auto-detect + +tts: + backend: "piper" # piper (Piper VITS — CPU ONNX, default) | f5 (F5-TTS voice cloning) + device: "auto" + default_voice: "default" # Piper voice name (see the Voices tab / voice-ref-curator) + +# Overnight voice training — the default voice is retrained once per night +# inside the start_hour → end_hour window, resuming from the latest checkpoint. +# Toggle/disable here (or in Settings → Voice Training); changes apply live. +training: + enabled: true + start_hour: 3 # window opens at 03:00 (0-23) + end_hour: 6 # window closes at 06:00 (0-23) + min_remaining_minutes: 60 # don't start a run if less window time remains + max_epochs: 6000 + batch_size: 32 + num_workers: 2 + accelerator: "gpu" # gpu | cpu + precision: "16-mixed" # 32 | 16-mixed | bf16-mixed + language: "de" # Piper language code + +audio: + sample_rate: 16000 + input_device: null # null = system default; or int device index + output_device: null + thinking_cue_delay_s: 2.5 # soft tick while Javis is still working; 0 = off + +wake_word: + enabled: true + model: "hey_jarvis" # built-in openWakeWord name, or a custom model: "rufus" -> ~/.javis/models/wakeword/rufus.onnx (see docs/WAKE_WORD.md) + threshold: 0.5 + vad_threshold: 0.5 # silero-vad probability threshold + silence_ms: 800 # end-of-utterance silence + +mcp: + servers_config: "mcp_servers.json" + approval_default: "ask" # ask | allow | deny (headless: "ask" is denied — no UI) + call_timeout_seconds: 30 + approval_timeout_seconds: 45 # auto-deny if the approval dialog isn't answered; 0 = wait + voice_approval: true # answer the approval prompt with "ja"/"nein" (needs STT + mic) + +api: + # HTTP API for javis-slave companion devices (plain HTTP — trusted LAN only). + enabled: false + host: "0.0.0.0" # bind address; 0.0.0.0 = all interfaces + port: 8765 + pairing_code_ttl_seconds: 300 # lifetime of a single-use pairing code + +agent: + # Persona preset. Choose one: default | smart_home | concise | verbose + preset: "default" + # Optional verbatim system prompt (overrides the preset above). + system_prompt_inline: null + # Optional path to a .txt file (highest priority — overrides everything else). + system_prompt_file: null + history_max_turns: 20 + # Maximum LLM <-> tool-call iterations per turn before giving up. + max_llm_iterations: 5 + # Spoken feedback for voice users (empty string = silent). + error_phrase: "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal." + approval_phrase: "Soll ich das ausführen? Sag ja oder nein." + +memory: + # "ollama" uses llm.base_url; "hermes" uses the remote Hermes API server. + provider: "ollama" + # Hermes API root (not the dashboard URL). Required when provider is hermes. + base_url: null + # Store API_SERVER_KEY in this environment variable; never put it in YAML. + api_key_env: "JAVIS_HERMES_API_KEY" + # Dedicated model for background fact extraction. + # null = use the same model as llm.model. + # Set to a smarter or reasoning model for higher-quality extraction without + # slowing down the main conversation (extraction runs in the background). + # Examples: "qwen2.5:14b", "qwen3:8b", "deepseek-r1:8b", "llama3.1:8b" + model: null + # Temperature for extraction calls. Low values (0.0–0.2) give more consistent + # JSON output. Reasoning models work well at 0.1. + temperature: 0.1 + # Context window for extraction. Much smaller than the main LLM is fine — + # extraction prompts are short. Reduce if memory calls are slow. + num_ctx: 4096 + # Timeout (seconds) for extraction calls. 120 is a safe default. + # Increase for reasoning models on slow hardware (deepseek-r1 can take 3–5 min on CPU). + # Set to null only to inherit llm.request_timeout_seconds (not recommended). + timeout_seconds: 120 + # Maximum number of new facts to extract per turn. + max_facts: 2 + # Maximum number of existing memories included in the extraction prompt + # (most-recently-updated first). Raise for larger memories, lower to keep + # extraction prompts short. + max_existing_for_prompt: 25 + +paths: + base_dir: null # null = ~/.javis diff --git a/docs/WAKE_WORD.md b/docs/WAKE_WORD.md new file mode 100644 index 0000000..d048e8f --- /dev/null +++ b/docs/WAKE_WORD.md @@ -0,0 +1,130 @@ +# Custom wake word ("Rufus") + +Javis uses [openWakeWord](https://github.com/dscripka/openWakeWord). The built-in +models are English phrases (`hey_jarvis`, `alexa`, `hey_mycroft`, `hey_rhasspy`, +`timer`, `weather`). Anything else — e.g. **Rufus** — is a small custom `.onnx` +classifier that you train once on synthetic speech (no recordings of your own +voice needed) and drop into Javis. Slaves download it from the master +automatically. + +## 1. How Javis loads wake-word models + +`wake_word.model` in `config.yaml` accepts three forms: + +| Value | Meaning | +|-------|---------| +| `hey_jarvis` | built-in openWakeWord model | +| `rufus` | custom model at `/models/wakeword/rufus.onnx` (default `~/.javis/models/wakeword/rufus.onnx`) | +| `D:/models/rufus.onnx` | explicit path to a custom model | + +The detector keys predictions by the file stem, so the file **must** be named +`.onnx`. Slaves receive the bare name (`rufus`) from the master and fetch +the file via `GET /api/v1/wake_word/model` on first use, caching it under +`/models/rufus.onnx`. + +## 2. Train the model + +Training runs on **Linux** (WSL2 on Windows works) or free Google Colab; it needs +a GPU for the ~10 min synthetic-speech step but finishes in well under an hour. + +### Option A — Colab (no setup) + +Open openWakeWord's +[automatic_model_training.ipynb](https://github.com/dscripka/openWakeWord/blob/main/notebooks/automatic_model_training.ipynb) +in Colab, run the cells top to bottom, and change only the config cell: + +```python +config["target_phrase"] = ["rufus", "hey rufus", "roofus", "hey roofus"] +config["model_name"] = "rufus" +config["n_samples"] = 5000 # positives; more = better, 20k+ for production +config["n_samples_val"] = 1000 +config["steps"] = 20000 +``` + +Download `my_custom_model/rufus.onnx` when it finishes. + +### Option B — locally (WSL2 / Linux, reproducible) + +```bash +# one-time environment +python3 -m venv ~/oww && source ~/oww/bin/activate +git clone https://github.com/dscripka/openwakeword && pip install -e ./openwakeword +git clone https://github.com/rhasspy/piper-sample-generator +wget -O piper-sample-generator/models/en_US-libritts_r-medium.pt \ + https://github.com/rhasspy/piper-sample-generator/releases/download/v2.0.0/en_US-libritts_r-medium.pt +pip install piper-phonemize webrtcvad mutagen torchinfo torchmetrics speechbrain \ + audiomentations torch-audiomentations acoustics pronouncing datasets deep-phonemizer + +# data: room impulse responses, background noise, pre-computed negative features +# (copy the "Download Data" cells from the notebook — ~3 GB total) + +# config +cp openwakeword/examples/custom_model.yml rufus.yml +# target_phrase: ["rufus", "hey rufus", "roofus", "hey roofus"] +# model_name: rufus +# n_samples: 5000 n_samples_val: 1000 steps: 20000 +# background_paths: ["./audioset_16k", "./fma"] +# false_positive_validation_data_path: validation_set_features.npy +# feature_data_files: {ACAV100M_sample: openwakeword_features_ACAV100M_2000_hrs_16bit.npy} + +python openwakeword/openwakeword/train.py --training_config rufus.yml --generate_clips +python openwakeword/openwakeword/train.py --training_config rufus.yml --augment_clips +python openwakeword/openwakeword/train.py --training_config rufus.yml --train_model +# → my_custom_model/rufus.onnx +``` + +### Tips that matter for a German household + +- **Spell out the German pronunciation.** The synthetic voices are English; add + phonetic variants to `target_phrase` so the model hears the "u" you actually + say: `"roofus"`, `"ruufus"`. Include both the bare name and `"hey "`. +- **Add adversarial negatives.** `custom_negative_phrases` in the YAML (e.g. + `["rufen", "ruf mal", "ruhe", "bus", "roof"]`) sharply cuts false wakes on + similar-sounding words. +- **Two-syllable names with a hard consonant** ("Rufus", "Jarvis") work far + better than short or vowel-heavy ones. +- Short names false-trigger more: expect to run `threshold: 0.6–0.7` rather + than the default 0.5. Tune in **Settings → Wake Word**; the Status tab shows + the live score. + +## 3. Install in Javis + +```powershell +mkdir $env:USERPROFILE\.javis\models\wakeword +copy rufus.onnx $env:USERPROFILE\.javis\models\wakeword\ +``` + +`config.yaml`: + +```yaml +wake_word: + model: rufus + threshold: 0.6 +``` + +Restart Javis. The log shows `openWakeWord loaded (…\rufus.onnx, key=rufus, …)`. +Paired slaves pick up the new name on their next config poll (≤ 60 s), download +the model from the master and reload the detector — nothing to copy by hand. +For a slave that cannot reach the master at that moment, place the file at +`~/.javis-slave/models/rufus.onnx` (or `%LOCALAPPDATA%\JavisSlave\models\`). + +## 4. Verify + +- Say the name at normal speaking distance: the wake chime plays and the + Status tab's *Wake score* bar jumps. Watch for a minute of normal + conversation without triggers; if it fires spuriously, raise the threshold or + retrain with more negatives. +- Quick offline check without Javis: + + ```powershell + python -c "from openwakeword.model import Model; m=Model(wakeword_models=[r'%USERPROFILE%\.javis\models\wakeword\rufus.onnx'], inference_framework='onnx'); print(m.predict_clip(r'test.wav'))" + ``` + +## Troubleshooting + +| Symptom | Cause / fix | +|---------|-------------| +| `wake word model 'rufus' is neither a built-in … nor a custom .onnx` | File missing or misnamed — it must be `rufus.onnx` in `models/wakeword/`. | +| Slave says `Wake-Word-Modell 'rufus' konnte nicht geladen werden` | Master unreachable during download or master itself has no custom file; check `GET http://master:8765/api/v1/wake_word/model` returns 200 with slave credentials. | +| Fires on TV / music | Raise `threshold`; retrain with more `background_paths` hours and `custom_negative_phrases`. | +| Misses when spoken quickly / far away | Add phrase variants, more `n_samples` (20k+), and set `augmentation_rounds` higher in the YAML. | diff --git a/duckduckgo-mcp/README.md b/duckduckgo-mcp/README.md new file mode 100644 index 0000000..a709776 --- /dev/null +++ b/duckduckgo-mcp/README.md @@ -0,0 +1,26 @@ +# duckduckgo-mcp + +MCP server that exposes DuckDuckGo search to Javis via three tools: + +- **`ddg_search`** — general web search (titles, URLs, snippets) +- **`ddg_news`** — recent news articles +- **`ddg_answers`** — instant answers (definitions, calculations, Wikipedia summaries) + +## Setup + +```bat +run.bat +``` + +The batch script creates a `.venv`, installs dependencies, and launches the server over stdio. + +## Registration in `mcp_servers.json` + +```json +"duckduckgo": { + "command": "d:\\Javis\\duckduckgo-mcp\\.venv\\Scripts\\duckduckgo-mcp.exe", + "args": [], + "transport": "stdio", + "approval": "allow" +} +``` diff --git a/duckduckgo-mcp/pyproject.toml b/duckduckgo-mcp/pyproject.toml new file mode 100644 index 0000000..b608f27 --- /dev/null +++ b/duckduckgo-mcp/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "duckduckgo-mcp" +version = "0.1.0" +description = "MCP server exposing DuckDuckGo web, news, and instant-answer search." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "duckduckgo-mcp" }] +keywords = ["mcp", "duckduckgo", "search", "llm"] +dependencies = [ + "mcp[cli]>=1.2", + "ddgs>=1.0", +] + +[project.scripts] +duckduckgo-mcp = "duckduckgo_mcp.server:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/duckduckgo_mcp"] diff --git a/duckduckgo-mcp/run.bat b/duckduckgo-mcp/run.bat new file mode 100644 index 0000000..c15984e --- /dev/null +++ b/duckduckgo-mcp/run.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +python -m duckduckgo_mcp %* diff --git a/duckduckgo-mcp/src/duckduckgo_mcp/__init__.py b/duckduckgo-mcp/src/duckduckgo_mcp/__init__.py new file mode 100644 index 0000000..b7ad0bf --- /dev/null +++ b/duckduckgo-mcp/src/duckduckgo_mcp/__init__.py @@ -0,0 +1 @@ +"""duckduckgo_mcp package.""" diff --git a/duckduckgo-mcp/src/duckduckgo_mcp/__main__.py b/duckduckgo-mcp/src/duckduckgo_mcp/__main__.py new file mode 100644 index 0000000..2bec294 --- /dev/null +++ b/duckduckgo-mcp/src/duckduckgo_mcp/__main__.py @@ -0,0 +1,5 @@ +"""Allow ``python -m duckduckgo_mcp`` to start the server.""" + +from .server import main + +main() diff --git a/duckduckgo-mcp/src/duckduckgo_mcp/server.py b/duckduckgo-mcp/src/duckduckgo_mcp/server.py new file mode 100644 index 0000000..45abd5f --- /dev/null +++ b/duckduckgo-mcp/src/duckduckgo_mcp/server.py @@ -0,0 +1,199 @@ +"""DuckDuckGo MCP server — web, news, and instant-answer search.""" + +from __future__ import annotations + +import logging +from typing import Any + +from mcp.server.fastmcp import FastMCP + +log = logging.getLogger("duckduckgo_mcp") + +mcp = FastMCP( + name="duckduckgo", + instructions=( + "Use these tools to search the web via DuckDuckGo. " + "Prefer `ddg_search` for general research, `ddg_news` for recent events, " + "and `ddg_answers` for quick factual lookups (calculations, definitions, etc.)." + ), +) + + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _get_ddgs(): + """Import DDGS lazily so import errors surface at call time. + + Tries the current package name ``ddgs`` first, then falls back to the + legacy ``duckduckgo_search`` name for environments that haven't migrated. + """ + try: + try: + from ddgs import DDGS # new package name (v1+) + except ImportError: + from duckduckgo_search import DDGS # legacy name # type: ignore[import-untyped] + return DDGS() + except ImportError as exc: + raise RuntimeError( + "Neither 'ddgs' nor 'duckduckgo-search' is installed. " + "Run: pip install ddgs" + ) from exc + + +def _fmt_web(results: list[dict[str, Any]]) -> str: + if not results: + return "No results found." + lines: list[str] = [] + for i, r in enumerate(results, 1): + title = r.get("title", "").strip() + url = r.get("href", "") + body = (r.get("body") or "").strip() + lines.append(f"{i}. **{title}**\n {url}\n {body}") + return "\n\n".join(lines) + + +def _fmt_news(results: list[dict[str, Any]]) -> str: + if not results: + return "No news results found." + lines: list[str] = [] + for i, r in enumerate(results, 1): + title = r.get("title", "").strip() + url = r.get("url", r.get("href", "")) + date = r.get("date", "") + source = r.get("source", "") + body = (r.get("body") or "").strip() + meta = " · ".join(filter(None, [source, date])) + lines.append(f"{i}. **{title}**{(' — ' + meta) if meta else ''}\n {url}\n {body}") + return "\n\n".join(lines) + + +def _fmt_answers(results: list[dict[str, Any]]) -> str: + if not results: + return "No instant answers found." + lines: list[str] = [] + for r in results: + text = (r.get("text") or r.get("answer") or "").strip() + url = r.get("url") or r.get("href") or "" + if text: + lines.append(text + (f"\n Source: {url}" if url else "")) + return "\n\n".join(lines) if lines else "No instant answers found." + + +# ── tools ───────────────────────────────────────────────────────────────────── + +@mcp.tool( + description=( + "Search the web using DuckDuckGo. " + "Returns titles, URLs, and short descriptions for the top results. " + "Use this for general research, how-to questions, finding documentation, " + "or any open-ended information lookup." + ) +) +def ddg_search( + query: str, + max_results: int = 10, + region: str = "wt-wt", + safesearch: str = "moderate", +) -> str: + """Web search via DuckDuckGo. + + Args: + query: The search query. + max_results: Number of results to return (1–30). + region: DuckDuckGo region code, e.g. 'wt-wt' (global), 'de-de', 'us-en'. + safesearch: 'on', 'moderate', or 'off'. + """ + max_results = max(1, min(max_results, 30)) + ddgs = _get_ddgs() + try: + results = ddgs.text( + query, + max_results=max_results, + region=region, + safesearch=safesearch, + ) + return _fmt_web(list(results)) + except Exception as exc: + log.error(f"ddg_search error: {exc}") + return f"Search failed: {exc}" + + +@mcp.tool( + description=( + "Search DuckDuckGo News for recent articles and breaking stories. " + "Use this for current events, news about people or companies, " + "or any topic where freshness matters." + ) +) +def ddg_news( + query: str, + max_results: int = 10, + region: str = "wt-wt", +) -> str: + """News search via DuckDuckGo. + + Args: + query: The news search query. + max_results: Number of articles to return (1–30). + region: DuckDuckGo region code, e.g. 'wt-wt' (global), 'de-de', 'us-en'. + """ + max_results = max(1, min(max_results, 30)) + ddgs = _get_ddgs() + try: + results = ddgs.news( + query, + max_results=max_results, + region=region, + ) + return _fmt_news(list(results)) + except Exception as exc: + log.error(f"ddg_news error: {exc}") + return f"News search failed: {exc}" + + +@mcp.tool( + description=( + "Look up a quick, focused answer for short factual questions — " + "definitions, calculations, unit conversions, 'who is X', 'what is Y', etc. " + "Returns the top 3 most relevant web results. " + "Prefer ddg_search for open-ended research." + ) +) +def ddg_answers(query: str) -> str: + """Quick factual lookup via DuckDuckGo. + + The DDG instant-answers API was removed upstream (duckduckgo-search >= 7 / + ddgs >= 1). This tool now returns the top 3 focused web results as the + best available substitute. + + Args: + query: A short, direct question or lookup term. + """ + ddgs = _get_ddgs() + try: + # Use native answers() if the installed library still provides it + # (kept for forward-compatibility if DDG ever restores the endpoint). + if hasattr(ddgs, "answers"): + try: + results = list(ddgs.answers(query)) + if results: + return _fmt_answers(results) + except Exception as exc: + log.debug(f"ddg native answers() failed ({exc}), falling back to text") + # Fallback: top-3 web results as a focused answer proxy + results = list(ddgs.text(query, max_results=3)) + return _fmt_web(results) if results else "No results found." + except Exception as exc: + log.error(f"ddg_answers error: {exc}") + return f"Answer lookup failed: {exc}" + + +# ── entry point ─────────────────────────────────────────────────────────────── + +def main() -> None: + logging.basicConfig(level=logging.INFO) + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/emby-mcp/.env.example b/emby-mcp/.env.example new file mode 100644 index 0000000..25f77d0 --- /dev/null +++ b/emby-mcp/.env.example @@ -0,0 +1,12 @@ +# Emby connection +EMBY_HOST=emby.spaeth-bayern.de +EMBY_API_KEY=c481ebd4812648f5a08f6557a0de4bc4 +EMBY_SCHEME=https +EMBY_PORT=443 +EMBY_TIMEOUT=10.0 + +# LAN transport (optional) - for remote MCP clients over the LAN +# EMBY_MCP_TRANSPORT=streamable-http +# EMBY_MCP_HOST=0.0.0.0 +# EMBY_MCP_ADVERTISED_HOST= +# EMBY_MCP_PORT=8080 \ No newline at end of file diff --git a/emby-mcp/README.md b/emby-mcp/README.md new file mode 100644 index 0000000..8db92cd --- /dev/null +++ b/emby-mcp/README.md @@ -0,0 +1,138 @@ +# emby-mcp + +MCP (Model Context Protocol) server that exposes an **Emby** media server to LLM clients. + +## Features + +Tools: +- `ping` — verify connectivity / return Emby system info +- `list_items` — browse the media library with optional filtering +- `search_items` — search for items by name or metadata +- `get_item` — get detailed information about a specific item +- `list_users` — list all users on the server +- `get_user` — get user details +- `list_sessions` — see who's currently connected / watching +- `system_info` — get server system information +- `playback_url` — get a direct stream URL for an item +- `playback_info` — get detailed playback/transcoding options +- `play_item` — send remote control commands to start playback +- `list_devices` — list registered devices + +Resources: +- `emby://library` — snapshot of the entire media library as JSON +- `emby://users` — snapshot of all users as JSON +- `emby://sessions` — snapshot of current sessions as JSON + +## Prerequisites + +- Python **3.10+** +- An Emby server (can be tested against: `emby.spaeth-bayern.de`) +- An **API Key** from your Emby dashboard + +## Install + +From this folder: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -e . +``` + +Or using `uv`: + +```powershell +uv venv +uv pip install -e . +``` + +## Configure + +Copy `.env.example` to `.env` and fill in your Emby details: + +```ini +EMBY_HOST=emby.spaeth-bayern.de +EMBY_API_KEY=your-api-key-here +EMBY_SCHEME=https +EMBY_TIMEOUT=10.0 +``` + +Get your API key from the Emby web dashboard: **Dashboard → Users → Manage Users → [your user] → Settings → API Keys**. + +## Run + +```powershell +.\run.bat +# or +python -m emby_mcp +``` + +### Inspect with MCP Inspector + +```powershell +uv run mcp dev src/emby_mcp/server.py +``` + +## Register in Claude Desktop / VS Code MCP config + +Add to your `mcp_servers.json`: + +```json +"emby": { + "command": "d:\\Javis\\emby-mcp\\.venv\\Scripts\\python.exe", + "args": ["-m", "emby_mcp"], + "env": { + "EMBY_HOST": "emby.spaeth-bayern.de", + "EMBY_API_KEY": "your-api-key-here" + }, + "transport": "stdio", + "approval": "allow" +} +``` + +## Usage Examples + +### Check connectivity +> "Is my Emby server running?" +```python +ping() +``` + +### Browse library +> "What movies do I have?" +```python +list_items(item_type="Movie") +``` + +### Search for content +> "Find all episodes of Stranger Things" +```python +search_items("Stranger Things", limit=50) +``` + +### Check active sessions +> "Who is watching something right now?" +```python +list_sessions() +``` + +### Get playback URL +> "Get me a stream URL for this movie" +```python +playback_url(item_id="...") +``` + +## Project Layout + +``` +emby-mcp/ + pyproject.toml + README.md + .env.example + run.bat + src/emby_mcp/ + __init__.py + __main__.py + config.py + client.py + server.py \ No newline at end of file diff --git a/emby-mcp/mcp-emby.example.json b/emby-mcp/mcp-emby.example.json new file mode 100644 index 0000000..f374445 --- /dev/null +++ b/emby-mcp/mcp-emby.example.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "emby": { + "command": "d:\\Javis\\emby-mcp\\.venv\\Scripts\\python.exe", + "args": ["-m", "emby_mcp"], + "env": { + "EMBY_HOST": "emby.spaeth-bayern.de", + "EMBY_API_KEY": "c481ebd4812648f5a08f6557a0de4bc4" + }, + "transport": "stdio", + "approval": "allow" + } + } +} \ No newline at end of file diff --git a/emby-mcp/pyproject.toml b/emby-mcp/pyproject.toml new file mode 100644 index 0000000..2fa5930 --- /dev/null +++ b/emby-mcp/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "emby-mcp" +version = "0.1.0" +description = "MCP server exposing an Emby media server to LLM clients." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "emby-mcp" }] +keywords = ["mcp", "emby", "media-server", "llm"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "mcp[cli]>=1.2", + "httpx>=0.27", + "pydantic>=2.7", + "pydantic-settings>=2.3", +] + +[project.scripts] +emby-mcp = "emby_mcp.server:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/emby_mcp"] \ No newline at end of file diff --git a/emby-mcp/run-lan.bat b/emby-mcp/run-lan.bat new file mode 100644 index 0000000..7786248 --- /dev/null +++ b/emby-mcp/run-lan.bat @@ -0,0 +1,20 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +set EMBY_MCP_TRANSPORT=streamable-http +set EMBY_MCP_HOST=0.0.0.0 +set EMBY_MCP_PORT=8080 +set EMBY_MCP_ADVERTISED_HOST=localhost + +python -m emby_mcp %* \ No newline at end of file diff --git a/emby-mcp/run.bat b/emby-mcp/run.bat new file mode 100644 index 0000000..5ee5b01 --- /dev/null +++ b/emby-mcp/run.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +python -m emby_mcp %* \ No newline at end of file diff --git a/emby-mcp/src/emby_mcp/__init__.py b/emby-mcp/src/emby_mcp/__init__.py new file mode 100644 index 0000000..193ae1c --- /dev/null +++ b/emby-mcp/src/emby_mcp/__init__.py @@ -0,0 +1,3 @@ +"""emby-mcp: MCP server for an Emby media server.""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/emby-mcp/src/emby_mcp/__main__.py b/emby-mcp/src/emby_mcp/__main__.py new file mode 100644 index 0000000..a6876bf --- /dev/null +++ b/emby-mcp/src/emby_mcp/__main__.py @@ -0,0 +1,4 @@ +from .server import main + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/emby-mcp/src/emby_mcp/client.py b/emby-mcp/src/emby_mcp/client.py new file mode 100644 index 0000000..0abbceb --- /dev/null +++ b/emby-mcp/src/emby_mcp/client.py @@ -0,0 +1,134 @@ +"""Async client for the Emby API.""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from .config import Settings + +log = logging.getLogger(__name__) + + +class EmbyClient: + """Thin async wrapper around Emby's REST API.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + self._http = httpx.AsyncClient( + base_url=settings.base_url, + headers={ + "X-Emby-Token": settings.api_key, + "Accept": "application/json", + }, + timeout=settings.timeout, + ) + + async def aclose(self) -> None: + await self._http.aclose() + + # ---- helpers ---------------------------------------------------------- + + async def _get(self, path: str, params: dict[str, Any] | None = None) -> Any: + r = await self._http.get(path, params=params) + r.raise_for_status() + return r.json() + + async def _post( + self, path: str, json: dict[str, Any] | None = None, params: dict[str, Any] | None = None + ) -> Any: + r = await self._http.post(path, json=json, params=params) + r.raise_for_status() + if r.content: + try: + return r.json() + except ValueError: + return None + return None + + async def _delete(self, path: str) -> Any: + r = await self._http.delete(path) + r.raise_for_status() + if r.content: + try: + return r.json() + except ValueError: + return None + return None + + # ---- API: Items (media library) --------------------------------------- + + async def list_items( + self, + parent_id: str | None = None, + include_related: bool = False, + recursive: bool = True, + sort_by: str | None = "Name", + sort_order: str = "Ascending", + limit: int | None = None, + ) -> dict[str, Any]: + """List items from the media library.""" + params: dict[str, Any] = { + "IncludeRelated": str(include_related).lower(), + "Recursive": str(recursive).lower(), + "SortBy": sort_by, + "SortOrder": sort_order, + } + if parent_id: + params["ParentId"] = parent_id + if limit: + params["Limit"] = limit + return await self._get("/Items", params=params) + + async def get_item(self, item_id: str) -> dict[str, Any]: + """Get details for a specific item.""" + return await self._get(f"/Items/{item_id}") + + # ---- API: Users ------------------------------------------------------- + + async def list_users(self) -> dict[str, Any]: + """List all users on the server.""" + return await self._get("/Users") + + async def get_user(self, user_id: str) -> dict[str, Any]: + """Get details for a specific user.""" + return await self._get(f"/Users/{user_id}") + + # ---- API: Sessions ---------------------------------------------------- + + async def list_sessions(self) -> list[dict[str, Any]]: + """List active sessions (connected devices/apps).""" + return await self._get("/Sessions") + + async def get_system_info(self) -> dict[str, Any]: + """Get server system info.""" + return await self._get("/System/Info") + + # ---- API: Playback ---------------------------------------------------- + + async def play_item( + self, item_id: str, user_id: str | None = None, autoplay: bool = True + ) -> Any: + """Start playback of an item.""" + params: dict[str, Any] = {"UserId": user_id or "null", "Autostart": autoplay} + return await self._post(f"/Items/{item_id}/Play", params=params) + + async def get_playback_info(self, item_id: str, user_id: str | None = None) -> dict[str, Any]: + """Get playback info for an item (transcoding options, etc.).""" + params = {} + if user_id: + params["UserId"] = user_id + return await self._get(f"/Items/{item_id}/PlaybackInfo", params=params) + + async def get_playback_url(self, item_id: str) -> dict[str, Any]: + """Get a stream URL for direct playback.""" + return await self._get(f"/Items/{item_id}/StreamUrl") + + # ---- API: Device Info ------------------------------------------------- + + async def list_devices(self) -> list[dict[str, Any]]: + """List registered devices.""" + r = await self._get("/Devices") + return r if isinstance(r, list) else [] \ No newline at end of file diff --git a/emby-mcp/src/emby_mcp/config.py b/emby-mcp/src/emby_mcp/config.py new file mode 100644 index 0000000..e9c1da0 --- /dev/null +++ b/emby-mcp/src/emby_mcp/config.py @@ -0,0 +1,40 @@ +"""Configuration loaded from environment variables / .env.""" + +from __future__ import annotations + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Settings for the emby-mcp server. + + Values are read from environment variables (prefix ``EMBY_``) or a + local ``.env`` file. + """ + + model_config = SettingsConfigDict( + env_prefix="EMBY_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + host: str = Field(..., description="IP or hostname of the Emby server.") + api_key: str = Field(..., description="Emby API key (from dashboard).") + scheme: str = Field("https", description="http or https.") + port: int | None = Field( + None, + description="Optional TCP port.", + ) + timeout: float = Field(10.0, description="HTTP timeout in seconds.") + + @property + def base_url(self) -> str: + if self.port is None: + return f"{self.scheme}://{self.host}" + return f"{self.scheme}://{self.host}:{self.port}" + + +def load_settings() -> Settings: + return Settings() # type: ignore[call-arg] \ No newline at end of file diff --git a/emby-mcp/src/emby_mcp/server.py b/emby-mcp/src/emby_mcp/server.py new file mode 100644 index 0000000..204e176 --- /dev/null +++ b/emby-mcp/src/emby_mcp/server.py @@ -0,0 +1,438 @@ +"""FastMCP server exposing an Emby media server.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any + +from mcp.server.fastmcp import FastMCP, Context +from pydantic import BaseModel, Field + +from .client import EmbyClient +from .config import load_settings + +log = logging.getLogger("emby_mcp") + + +# ---- models -------------------------------------------------------------- + + +class ItemSummary(BaseModel): + id: str + name: str + type: str | None = None + parent_id: str | None = Field(default=None, alias="ParentId") + series_name: str | None = Field(default=None, alias="SeriesName") + season_number: int | None = Field(default=None, alias="SeasonNumber") + episode_number: int | None = Field(default=None, alias="EpisodeNumber") + year: int | None = None + runtime_minutes: int | None = Field(default=None, alias="RunTimeNumeric") + size_mb: float | None = Field(default=None, alias="Size") + image_url: str | None = Field(default=None, alias="ImageTags") + + model_config = {"populate_by_name": True} + + +class UserSummary(BaseModel): + id: str + name: str + is_admin: bool = False + has_authorization: bool = True + + model_config = {"populate_by_name": True} + + +class SessionInfo(BaseModel): + user_id: str | None = Field(default=None, alias="UserId") + user_name: str | None = Field(default=None, alias="UserName") + client_name: str | None = Field(default=None, alias="ClientName") + device_name: str | None = Field(default=None, alias="DeviceName") + app_id: str | None = Field(default=None, alias="AppId") + device_type: str | None = Field(default=None, alias="DeviceType") + + model_config = {"populate_by_name": True} + + +class PlaybackInfo(BaseModel): + playable_url: str | None = None + transcoding_url: str | None = None + can_transcode: bool = False + audio_codec: str | None = None + video_codec: str | None = None + + model_config = {"populate_by_name": True} + + +# ---- lifespan ------------------------------------------------------------ + +# Shared client for all operations (tools and resources) +_shared_client: EmbyClient | None = None + + +@dataclass +class AppContext: + emby: EmbyClient + + +@asynccontextmanager +async def lifespan(_server: FastMCP) -> AsyncIterator[AppContext]: + global _shared_client + settings = load_settings() + client = EmbyClient(settings) + _shared_client = client + try: + yield AppContext(emby=client) + finally: + _shared_client = None + await client.aclose() + + +# ---- MCP server ---------------------------------------------------------- + +mcp = FastMCP( + "emby-mcp", + instructions=( + "Tools for inspecting and controlling an Emby media server.\n\n" + "AVAILABLE TOOLS SUMMARY:\n" + " Library: list_items, get_item, search_items\n" + " Users: list_users, get_user\n" + " Sessions:list_sessions, system_info\n" + " Playback:play_item, playback_url, playback_info\n\n" + "COMMON WORKFLOWS:\n" + " 'What movies do I have?' → list_items(type='Movie')\n" + " 'Play Stranger Things' → search_items('Stranger Things') → play_item(id)\n" + " 'Who is connected?' → list_sessions()\n" + ), + lifespan=lifespan, +) + + +# ---- helpers ------------------------------------------------------------- + + +def _emby(ctx: Context) -> EmbyClient: + app: AppContext = ctx.request_context.lifespan_context # type: ignore[assignment] + return app.emby + + +def _item_to_summary(item: dict[str, Any]) -> ItemSummary: + """Convert an Emby item dict to a summary.""" + return ItemSummary( + id=str(item.get("Id", "")), + name=item.get("Name"), + type=item.get("Type"), + parent_id=item.get("ParentId"), + series_name=item.get("SeriesName"), + season_number=item.get("SeasonNumber"), + episode_number=item.get("EpisodeNumber"), + year=item.get("ProductionYear") or item.get("Year"), + runtime_minutes=item.get("RunTimeNumeric"), + size_mb=item.get("Size") / (1024 * 1024) if item.get("Size") else None, + ) + + +# ---- tools --------------------------------------------------------------- + +@mcp.tool() +async def ping(ctx: Context) -> dict[str, Any]: + """Verify connectivity to the Emby server and return system info.""" + emby = _emby(ctx) + try: + info = await emby.get_system_info() + return { + "ok": True, + "server_name": info.get("ServerName"), + "version": info.get("Version"), + "id": info.get("Id"), + "date_created": info.get("dateCreated"), + } + except Exception as exc: + return {"ok": False, "error": str(exc)} + + +@mcp.tool() +async def list_items( + ctx: Context, + parent_id: str | None = None, + limit: int = 50, + sort_by: str | None = "Name", +) -> list[ItemSummary]: + """List items from the Emby media library. + + Args: + parent_id: Parent folder ID to browse within (optional). + limit: Maximum number of items to return (default 50). + sort_by: Sort field name, e.g. 'Name', 'DateCreated', 'Rating' (default 'Name'). + """ + emby = _emby(ctx) + raw = await emby.list_items(parent_id=parent_id, limit=limit, sort_by=sort_by) + items = raw.get("Items", []) if isinstance(raw, dict) else [] + + return [_item_to_summary(i) for i in items if isinstance(i, dict)] + + +@mcp.tool() +async def search_items(ctx: Context, query: str, limit: int = 20) -> list[ItemSummary]: + """Search for media items by name. + + Note: Currently performs a client-side filter on item names. For full-text search, + use the Emby web interface or provide specific item types and parent folders. + + Args: + query: Text to search for in item names. + limit: Maximum number of results (default 20). + """ + emby = _emby(ctx) + # Fetch a larger batch to filter on client side (search not natively supported by client) + raw = await emby.list_items(limit=limit * 3) + items = raw.get("Items", []) if isinstance(raw, dict) else [] + + # Simple client-side filter on Name field + query_lower = query.lower() + filtered = [ + i for i in items + if isinstance(i, dict) and query_lower in (i.get("Name") or "").lower() + ][:limit] + + return [_item_to_summary(i) for i in filtered] + + +@mcp.tool() +async def get_item(ctx: Context, item_id: str) -> dict[str, Any]: + """Get detailed information about a specific media item. + + Args: + item_id: The Emby item ID (UUID format). + """ + emby = _emby(ctx) + return await emby.get_item(item_id) + + +@mcp.tool() +async def list_users(ctx: Context) -> list[UserSummary]: + """List all users on the Emby server.""" + emby = _emby(ctx) + raw = await emby.list_users() + users = raw if isinstance(raw, list) else [] + + return [ + UserSummary( + id=str(u.get("Id", "")), + name=u.get("Name"), + is_admin=u.get("IsAdmin", False), + has_authorization=u.get("HasAuthorization", True), + ) + for u in users if isinstance(u, dict) + ] + + +@mcp.tool() +async def get_user(ctx: Context, user_id: str) -> dict[str, Any]: + """Get detailed information about a specific user. + + Args: + user_id: The Emby user ID. + """ + emby = _emby(ctx) + return await emby.get_user(user_id) + + +@mcp.tool() +async def list_sessions(ctx: Context) -> list[SessionInfo]: + """List all active sessions (connected devices/apps).""" + emby = _emby(ctx) + raw = await emby.list_sessions() + + return [ + SessionInfo( + user_id=s.get("UserId"), + user_name=s.get("UserName"), + client_name=s.get("ClientName"), + device_name=s.get("DeviceName"), + app_id=s.get("AppId"), + device_type=s.get("DeviceType"), + ) + for s in raw if isinstance(s, dict) + ] + + +@mcp.tool() +async def system_info(ctx: Context) -> dict[str, Any]: + """Get Emby server system information.""" + emby = _emby(ctx) + return await emby.get_system_info() + + +@mcp.tool() +async def playback_url( + ctx: Context, item_id: str, user_id: str | None = None, format: str = "mp4" +) -> dict[str, Any]: + """Get a direct stream URL for playing media. + + Args: + item_id: The Emby item ID to play. + user_id: Optional user ID for access control. + format: Output format (default mp4). + + Note: This returns a temporary URL that may expire. Use playback_info first + if you need transcoding options. + """ + emby = _emby(ctx) + try: + result = await emby.get_playback_url(item_id) + return {"url": result, "item_id": item_id} + except Exception as exc: + return {"error": str(exc), "item_id": item_id} + + +@mcp.tool() +async def playback_info( + ctx: Context, item_id: str, user_id: str | None = None +) -> dict[str, Any]: + """Get detailed playback information for an item. + + Returns transcoding options, compatible codecs, and stream URLs + for direct playback or transcoding. + + Args: + item_id: The Emby item ID. + user_id: Optional user ID for access control. + """ + emby = _emby(ctx) + try: + info = await emby.get_playback_info(item_id, user_id) + + # Extract relevant playback information + result: dict[str, Any] = { + "item_id": item_id, + "can_transcode": info.get("CanTranscode", False), + "media_sources": [], + } + + media_sources = info.get("MediaSources", []) + for source in media_sources: + if isinstance(source, dict): + result["media_sources"].append({ + "id": source.get("Id"), + "protocol": source.get("Protocol"), + "container": source.get("Container"), + "video_codec": source.get("VideoCodec"), + "audio_codec": source.get("AudioCodec"), + "direct_path": source.get("DirectPath"), + }) + + return result + except Exception as exc: + return {"error": str(exc), "item_id": item_id} + + +@mcp.tool() +async def play_item( + ctx: Context, + item_id: str, + user_id: str | None = None, + autoplay: bool = True, +) -> dict[str, Any]: + """Start playback of a media item. + + Args: + item_id: The Emby item ID to play. + user_id: Optional user ID for access control. + autoplay: Whether to start playback immediately (default True). + + Note: This sends a remote control command to connected sessions. + """ + emby = _emby(ctx) + try: + result = await emby.play_item(item_id, user_id, autoplay) + return {"ok": True, "item_id": item_id, "result": result} + except Exception as exc: + return {"ok": False, "error": str(exc), "item_id": item_id} + + +@mcp.tool() +async def list_devices(ctx: Context) -> list[dict[str, Any]]: + """List registered devices connected to the Emby server.""" + emby = _emby(ctx) + return await emby.list_devices() + + +# ---- resources ----------------------------------------------------------- + +@mcp.resource("emby://library") +async def resource_library() -> str: + """Snapshot of the entire media library as JSON.""" + import json + + emby = _shared_client + if emby is None: + raise RuntimeError("Emby client not initialized (server may be shutting down)") + + # Get all items without limit for a full snapshot + raw = await emby.list_items(limit=1000) + items = raw.get("Items", []) if isinstance(raw, dict) else [] + summaries = [_item_to_summary(i).model_dump() for i in items if isinstance(i, dict)] + return json.dumps(summaries, indent=2, default=str) + + +@mcp.resource("emby://users") +async def resource_users() -> str: + """Snapshot of all users as JSON.""" + import json + + emby = _shared_client + if emby is None: + raise RuntimeError("Emby client not initialized (server may be shutting down)") + + raw = await emby.list_users() + users = raw if isinstance(raw, list) else [] + result = [ + { + "id": str(u.get("Id", "")), + "name": u.get("Name"), + "is_admin": u.get("IsAdmin", False), + } + for u in users if isinstance(u, dict) + ] + return json.dumps(result, indent=2) + + +@mcp.resource("emby://sessions") +async def resource_sessions() -> str: + """Snapshot of current sessions as JSON.""" + import json + + emby = _shared_client + if emby is None: + raise RuntimeError("Emby client not initialized (server may be shutting down)") + + raw = await emby.list_sessions() + result = [ + { + "user_name": s.get("UserName"), + "client_name": s.get("ClientName"), + "device_name": s.get("DeviceName"), + "app_id": s.get("AppId"), + } + for s in raw if isinstance(s, dict) + ] + return json.dumps(result, indent=2) + + +# ---- entry point --------------------------------------------------------- + +def main() -> None: + import logging as _logging + + _logging.basicConfig( + level=_logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + mcp.run() # stdio transport for local MCP hosts + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/homey-mcp/.env.example b/homey-mcp/.env.example new file mode 100644 index 0000000..17629c7 --- /dev/null +++ b/homey-mcp/.env.example @@ -0,0 +1,16 @@ +# Homey connection +HOMEY_HOST=192.168.178.201 +HOMEY_TOKEN=4a6d5464-2e25-4fbe-bd2a-f9c4902c90e5:ac38c7fb-3131-4af5-997e-fd0e22c931c3:a3445d0d939932384f5de2594dae1857f5d2a53f +HOMEY_SCHEME=http +HOMEY_PORT=4859 +HOMEY_TIMEOUT=10 +HOMEY_EVENT_BUFFER=200 +HOMEY_REALTIME=true + +# LAN transport (optional) - only used by run-lan.bat / HOMEY_MCP_TRANSPORT=streamable-http +# HOMEY_MCP_TRANSPORT=streamable-http +# HOMEY_MCP_HOST=0.0.0.0 # uvicorn bind address (0.0.0.0 for all interfaces) +# HOMEY_MCP_ADVERTISED_HOST=10.10.10.50 # advertised host for MCP Host header validation +# HOMEY_MCP_PORT=8080 +# HOMEY_MCP_AUTH_MODE=none +# HOMEY_MCP_AUTH_TOKEN=replace-with-a-long-random-secret diff --git a/homey-mcp/.gitignore b/homey-mcp/.gitignore new file mode 100644 index 0000000..93d0a12 --- /dev/null +++ b/homey-mcp/.gitignore @@ -0,0 +1,22 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +.venv/ +venv/ + +# Env +.env +.env.local + +# Tooling +.mypy_cache/ +.pytest_cache/ +.ruff_cache/ + +# IDE +.vscode/ +.idea/ diff --git a/homey-mcp/README.md b/homey-mcp/README.md new file mode 100644 index 0000000..a7bb4bf --- /dev/null +++ b/homey-mcp/README.md @@ -0,0 +1,200 @@ +# homey-mcp + +An MCP (Model Context Protocol) server that exposes a local **Homey Pro** instance to LLM clients (Claude Desktop, VS Code / GitHub Copilot Chat, MCP Inspector, …). + +It talks to Homey's local HTTP Web API (`/api/manager/...`) using a Personal Access Token, and optionally subscribes to Socket.IO realtime events. + +## Features + +Tools: + +- `ping` — verify connectivity / return Homey system info +- `list_devices(zone_id?)` — devices with current capability values +- `get_device(device_id)` — full device object +- `list_zones` — rooms / areas +- `set_capability(device_id, capability, value)` — set `onoff`, `dim`, `target_temperature`, … +- `list_flows` — all flows +- `trigger_flow(flow_id)` — start a flow +- `list_recent_events(limit, kind?)` — recent realtime events from the in-memory buffer + +Resources: + +- `homey://devices` +- `homey://zones` +- `homey://events/recent` + +Transport: **stdio** (default, for local LLM hosts like Javis) or **streamable-http / sse** (for remote MCP clients over the LAN, e.g. the Odysseus AI helper) — see [Run on the LAN](#run-on-the-lan-remote-mcp-clients) below. + +## Prerequisites + +- Python **3.10+** +- A Homey Pro on the local network +- A **Personal Access Token** for the Homey Web API — create one at the Homey Developer Tools: (My Homey → API Keys / Personal Access Token). + +## Install + +From this folder: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -e . +``` + +`uv` works equivalently: + +```powershell +uv venv +uv pip install -e . +``` + +## Configure + +Copy `.env.example` to `.env` and fill in: + +```ini +HOMEY_HOST=192.168.1.10 +HOMEY_TOKEN=eyJhbGciOi...your PAT... +HOMEY_SCHEME=http +HOMEY_PORT=4859 +HOMEY_TIMEOUT=10 +HOMEY_EVENT_BUFFER=200 +HOMEY_REALTIME=true +``` + +## Run + +```powershell +homey-mcp +# or +python -m homey_mcp +# or +.\run.bat +``` + +### Inspect with the MCP Inspector + +```powershell +uv run mcp dev src/homey_mcp/server.py +# or +npx -y @modelcontextprotocol/inspector homey-mcp +``` + +### Run on the LAN (remote MCP clients) + +To let a remote MCP client on the same network (e.g. **Odysseus**) connect over +HTTP instead of stdio, set the transport env vars and run `run-lan.bat` +(or set them yourself and run `homey-mcp` / `python -m homey_mcp`): + +```ini +HOMEY_MCP_TRANSPORT=streamable-http # or: sse +HOMEY_MCP_HOST=0.0.0.0 # bind address (or specific LAN IP) +HOMEY_MCP_ADVERTISED_HOST=10.10.10.50 # advertised host for MCP validation (required for 0.0.0.0 bind) +HOMEY_MCP_PORT=8080 +HOMEY_MCP_AUTH_MODE=none # compatible trusted-LAN default +HOMEY_MCP_AUTH_TOKEN= +``` + +```powershell +.\run-lan.bat +``` + +The remote client connects to `http://:8080/mcp` +(streamable-http) or `/sse`. Streamable HTTP clients that only accept a base +URL may use `http://:8080/`; MCP traffic at `/` is forwarded +to `/mcp`, while an ordinary browser GET returns service discovery. + +**Important for MCP transport over LAN:** The MCP SDK validates that clients +send the correct `Host` header. If you bind to `0.0.0.0`, you **must** set +`HOMEY_MCP_ADVERTISED_HOST` to your actual server IP (e.g., `10.10.10.50`). +If you instead bind to a specific IP, omit `HOMEY_MCP_ADVERTISED_HOST` and it +will default to the bind address. + +`HOMEY_MCP_AUTH_MODE=none` is the compatible mode for OAuth-negotiating MCP +clients such as Odysseus, because this server is not an OAuth authorization +server. Use it only on a trusted, firewall-restricted LAN. Set the mode to +`bearer` only when the MCP client can be configured to send +`Authorization: Bearer ` directly; OAuth discovery and +dynamic registration are not provided. Generate a token with +`python -c "import secrets; print(secrets.token_urlsafe(32))"`. + +When `HOMEY_MCP_AUTH_TOKEN` is set, the OpenAPI REST routes always require it, +regardless of the MCP auth mode. + +This is independent of Javis's own connection: Javis keeps talking to +homey-mcp over stdio via `mcp_servers.json` as before — run the LAN instance +as a second, separate process for other clients. + +#### Odysseus API integration + +If Odysseus exposes integrations through its `app_api` tool rather than a +native MCP client, import this server's OpenAPI document: + +- Schema URL: `http://:8080/openapi.json` +- Authentication: Bearer token +- Token: the value of `HOMEY_MCP_AUTH_TOKEN` + +The API provides reads at `/api/system`, `/api/zones`, and `/api/devices`, +plus writes at `/api/devices/{device_id}/capabilities/{capability}` and +`/api/zones/control`. + +Odysseus must invoke its registered `app_api` tool as a structured tool call. +It must not print or execute Python such as `api_call(...)`; `api_call` is not +a Python function and is not the registered tool name. Use this integration +instruction in Odysseus: + +```text +For Homey requests, always call the app_api tool with integration="Homey". +Never write api_call(...) or place tool calls in Python/code blocks. +Use GET /api/system to check connectivity, GET /api/zones and GET /api/devices +to discover IDs, PUT /api/devices/{device_id}/capabilities/{capability} with +{"value": ...} for one device, and POST /api/zones/control with +{"zone": "...", "capability": "onoff", "value": true|false} for a room. +After the tool returns, summarize the actual result. Never claim success before +receiving a successful tool response. +``` + +### Use from Claude Desktop / VS Code + +Add to your MCP client config (path varies per client): + +```json +{ + "mcpServers": { + "homey": { + "command": "homey-mcp", + "env": { + "HOMEY_HOST": "192.168.1.10", + "HOMEY_TOKEN": "your-personal-access-token" + } + } + } +} +``` + +If `homey-mcp` is not on PATH, use the full venv path, e.g. `C:\\local\\Code\\PRIVAT\\homey-mcp\\.venv\\Scripts\\homey-mcp.exe`. + +## Notes & limitations + +- Local API only (no cloud OAuth). +- LAN MCP transport authentication is controlled by `HOMEY_MCP_AUTH_MODE`. + Never expose `none` mode beyond a trusted, firewall-restricted LAN. +- Realtime events are best-effort — if the Socket.IO connection fails the REST tools still work. Set `HOMEY_REALTIME=false` to disable. +- No admin operations (zone/user/app management) are exposed. +- Capability values are typed as `bool | int | float | str`; the model picks based on the capability (`onoff` → bool, `dim` → 0..1 float, etc.). + +## Project layout + +``` +homey-mcp/ + pyproject.toml + README.md + .env.example + run.bat + src/homey_mcp/ + __init__.py + __main__.py + config.py + homey_client.py + server.py +``` diff --git a/homey-mcp/UBUNTU_SETUP.md b/homey-mcp/UBUNTU_SETUP.md new file mode 100644 index 0000000..553a02f --- /dev/null +++ b/homey-mcp/UBUNTU_SETUP.md @@ -0,0 +1,137 @@ +# Homey MCP Server - Ubuntu Setup Guide + +## Quick Start + +### 1. Run the setup script +```bash +cd homey-mcp +chmod +x setup-ubuntu.sh run.sh run-lan.sh +./setup-ubuntu.sh +``` + +### 2. Configure your credentials +Edit the `.env` file with your Homey Pro credentials: +```bash +nano .env +``` + +Required fields: +- `HOMEY_TOKEN` - Your Homey Pro personal access token +- `HOMEY_HOST` - Your Homey Pro hub IP (e.g., `192.168.1.100`) +- `HOMEY_SCHEME` / `HOMEY_PORT` - Local API protocol and port + +Optional for LAN mode: +- `HOMEY_MCP_AUTH_MODE` - `none` for trusted-LAN compatibility, or `bearer` +- `HOMEY_MCP_AUTH_TOKEN` - Protects REST and, in `bearer` mode, MCP requests +- `HOMEY_MCP_ADVERTISED_HOST` - Set this to your server's LAN IP (e.g., `10.10.10.50`). Required if using `HOMEY_MCP_HOST=0.0.0.0`. + +### 3. Start the server + +**For remote Odysseus client (LAN mode):** +```bash +./run-lan.sh +``` + +**For local stdio mode (Javis integration):** +```bash +./run.sh +``` + +## Manual Installation + +If you prefer to set up manually: + +```bash +# Install Python (if not installed) +sudo apt install python3.10 python3.venv build-essential libffi-dev libssl-dev + +# Create virtual environment +cd homey-mcp +python3 -m venv .venv +source .venv/bin/activate + +# Install dependencies +pip install --upgrade pip +pip install -e . +pip install uvicorn + +# Configure +cp .env.example .env +nano .env # Edit with your credentials + +# Run +HOMEY_MCP_TRANSPORT=streamable-http HOMEY_MCP_HOST=0.0.0.0 HOMEY_MCP_PORT=8080 \ + python -m homey_mcp +``` + +## Persistent Operation with systemd + +Install the service file for automatic startup: + +```bash +# Copy service file +sudo cp homey-mcp.service /etc/systemd/system/ + +# Reload systemd +sudo systemctl daemon-reload + +# Enable and start (replace $USER with your username) +sudo systemctl enable --now homey-mcp@$USER.service + +# Check status +sudo journalctl -u homey-mcp@$USER.service -f +``` + +## Connecting Odysseus + +To connect Odysseus as an MCP client, use the streamable-http transport: + +1. **Important:** Before starting the server, set `HOMEY_MCP_ADVERTISED_HOST` in `.env` to your Ubuntu server's LAN IP address (e.g., `10.10.10.50`). This is required for MCP Host header validation. +2. Ensure homey-mcp is running on port 8080 (or your configured port) +3. Configure Odysseus with the MCP endpoint: + ``` + URL: http://:8080/mcp + Transport: streamable-http + Authentication: none + ``` + +If you see a "421 Misdirected Request" error, your `HOMEY_MCP_ADVERTISED_HOST` is not set correctly; it must match the IP that Odysseus uses to connect. + +Use `HOMEY_MCP_AUTH_MODE=none` only on a trusted, firewall-restricted LAN. +This avoids an OAuth discovery flow, which homey-mcp does not implement. If +Odysseus supports a manually configured static Authorization header, you may +instead set `HOMEY_MCP_AUTH_MODE=bearer` and configure the value of +`HOMEY_MCP_AUTH_TOKEN` as its bearer token. + +For Odysseus OpenAPI/`app_api` integration, import +`http://:8080/openapi.json` and configure +`HOMEY_MCP_AUTH_TOKEN` as a bearer token. REST routes remain protected even +when MCP authentication is disabled. + +## Troubleshooting + +### Virtual environment issues +```bash +rm -rf .venv +python3 -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +### Port already in use +```bash +# Check what's using the port +sudo lsof -i :8080 + +# Use a different port +HOMEY_MCP_PORT=8081 ./run-lan.sh +``` + +### Permission denied +```bash +chmod +x run.sh run-lan.sh setup-ubuntu.sh +``` + +### Check logs +```bash +sudo journalctl -u homey-mcp@$USER.service -f --since "1 hour ago" \ No newline at end of file diff --git a/homey-mcp/homey-mcp.service b/homey-mcp/homey-mcp.service new file mode 100644 index 0000000..790bd75 --- /dev/null +++ b/homey-mcp/homey-mcp.service @@ -0,0 +1,27 @@ +[Unit] +Description=Homey MCP Server (LAN mode for Odysseus) +After=network.target +Wants=network.target + +[Service] +Type=simple +User=%i +WorkingDirectory=%h/homey-mcp +ExecStart=%h/homey-mcp/run-lan.sh +Environment=HOMEY_MCP_TRANSPORT=streamable-http +Environment=HOMEY_MCP_HOST=0.0.0.0 +Environment=HOMEY_MCP_PORT=8080 +EnvironmentFile=%h/homey-mcp/.env +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=%h/homey-mcp +ProtectHome=read-only + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/homey-mcp/mcp-odysseus.example.json b/homey-mcp/mcp-odysseus.example.json new file mode 100644 index 0000000..3061094 --- /dev/null +++ b/homey-mcp/mcp-odysseus.example.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "homey": { + "url": "http://:8080/mcp", + "headers": { + "Authorization": "Bearer " + }, + "transport": "streamable-http" + } + } +} \ No newline at end of file diff --git a/homey-mcp/pyproject.toml b/homey-mcp/pyproject.toml new file mode 100644 index 0000000..ccf655c --- /dev/null +++ b/homey-mcp/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "homey-mcp" +version = "0.1.0" +description = "MCP server exposing a local Homey Pro instance to LLM clients." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "homey-mcp" }] +keywords = ["mcp", "homey", "llm", "smart-home"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "mcp[cli]>=1.2", + "httpx>=0.27", + "python-socketio[asyncio_client]>=5.11", + "pydantic>=2.7", + "pydantic-settings>=2.3", + "python-dotenv>=1.0", + "uvicorn[standard]>=0.29", + "starlette>=0.37", +] + +[project.scripts] +homey-mcp = "homey_mcp.server:main" + +[project.urls] +Homepage = "https://api.developer.homey.app/" + +[tool.hatch.build.targets.wheel] +packages = ["src/homey_mcp"] diff --git a/homey-mcp/run-lan.bat b/homey-mcp/run-lan.bat new file mode 100644 index 0000000..4695ea9 --- /dev/null +++ b/homey-mcp/run-lan.bat @@ -0,0 +1,22 @@ +@echo off +rem Runs homey-mcp as a LAN-reachable HTTP service (streamable-http transport) +rem for remote MCP clients (e.g. the Odysseus AI helper), instead of the +rem stdio transport Javis uses locally. +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +set HOMEY_MCP_TRANSPORT=streamable-http +if "%HOMEY_MCP_HOST%"=="" set HOMEY_MCP_HOST=0.0.0.0 +if "%HOMEY_MCP_PORT%"=="" set HOMEY_MCP_PORT=8080 + +python -m homey_mcp %* diff --git a/homey-mcp/run-lan.sh b/homey-mcp/run-lan.sh new file mode 100644 index 0000000..2b0a828 --- /dev/null +++ b/homey-mcp/run-lan.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run homey-mcp as a LAN-reachable HTTP MCP server for remote clients (e.g., Odysseus). +# Usage: ./run-lan.sh +# +# Environment variables (with defaults): +# HOMEY_MCP_TRANSPORT - streamable-http (default) or sse +# HOMEY_MCP_HOST - 0.0.0.0 (default) +# HOMEY_MCP_PORT - 8080 (default) +# HOMEY_MCP_AUTH_MODE - none (compatible default) or bearer +# HOMEY_MCP_AUTH_TOKEN - protects REST; also MCP when auth mode is bearer +set -euo pipefail +cd "$(dirname "$0")" + +# Create virtual environment if it doesn't exist +if [ ! -d ".venv" ]; then + echo "Creating virtual environment..." + python3 -m venv .venv +fi + +source .venv/bin/activate + +# Upgrade pip and install dependencies +python -m pip install --upgrade pip >/dev/null 2>&1 +pip install -e . >/dev/null 2>&1 + +# Set transport to streamable-http (recommended) or sse +export HOMEY_MCP_TRANSPORT="${HOMEY_MCP_TRANSPORT:-streamable-http}" +export HOMEY_MCP_HOST="${HOMEY_MCP_HOST:-0.0.0.0}" +export HOMEY_MCP_PORT="${HOMEY_MCP_PORT:-8080}" + +echo "Starting homey-mcp (LAN mode) on http://${HOMEY_MCP_HOST}:${HOMEY_MCP_PORT}" +echo "Transport: ${HOMEY_MCP_TRANSPORT}" + +exec python -m homey_mcp "$@" \ No newline at end of file diff --git a/homey-mcp/run.bat b/homey-mcp/run.bat new file mode 100644 index 0000000..873ccd9 --- /dev/null +++ b/homey-mcp/run.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +python -m homey_mcp %* diff --git a/homey-mcp/run.sh b/homey-mcp/run.sh new file mode 100644 index 0000000..81a0fa9 --- /dev/null +++ b/homey-mcp/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run homey-mcp as a local MCP server via stdio transport. +# Usage: ./run.sh +set -euo pipefail +cd "$(dirname "$0")" + +# Create virtual environment if it doesn't exist +if [ ! -d ".venv" ]; then + echo "Creating virtual environment..." + python3 -m venv .venv +fi + +source .venv/bin/activate + +# Upgrade pip and install dependencies +python -m pip install --upgrade pip >/dev/null 2>&1 +pip install -e . >/dev/null 2>&1 + +# Run with stdio transport (default for local MCP clients) +export HOMEY_MCP_TRANSPORT="${HOMEY_MCP_TRANSPORT:-stdio}" + +exec python -m homey_mcp "$@" \ No newline at end of file diff --git a/homey-mcp/setup-ubuntu.sh b/homey-mcp/setup-ubuntu.sh new file mode 100644 index 0000000..67c7411 --- /dev/null +++ b/homey-mcp/setup-ubuntu.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# One-time setup script for Ubuntu environment. +# Installs system dependencies, sets up the Python venv, and prepares .env. +set -euo pipefail +cd "$(dirname "$0")" + +echo "=== Homey MCP Server Ubuntu Setup ===" + +# 1. Check Python version +PYTHON_CMD="" +if command -v python3 &>/dev/null; then + PYTHON_CMD="python3" +elif command -v python &>/dev/null; then + PYTHON_CMD="python" +else + echo "ERROR: Python not found. Install Python 3.10+ first." + echo " Ubuntu: sudo apt install python3.10 python3.venv" + exit 1 +fi + +PY_VER=$($PYTHON_CMD -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') +echo "Found Python $PY_VER" + +# 2. Install system dependencies if needed +if ! command -v uvicorn &>/dev/null 2>&1; then + echo "Installing system dependencies..." + sudo apt-get update -qq + sudo apt-get install -y python3-pip python3-dev python3-venv build-essential libffi-dev libssl-dev +fi + +# 3. Create virtual environment if it doesn't exist +if [ ! -d ".venv" ]; then + echo "Creating virtual environment..." + $PYTHON_CMD -m venv .venv +fi + +source .venv/bin/activate + +# 4. Install/update Python dependencies +echo "Installing Python dependencies..." +$PYTHON_CMD -m pip install --upgrade pip >/dev/null 2>&1 +pip install -e . >/dev/null 2>&1 +pip install uvicorn >/dev/null 2>&1 + +# 5. Create .env file from example if it doesn't exist +if [ ! -f ".env" ]; then + if [ -f ".env.example" ]; then + echo "Creating .env from .env.example (please edit with your credentials)..." + cp .env.example .env + else + echo "Creating minimal .env file..." + cat > .env << 'EOF' +# Homey Pro API configuration +HOMEY_HOST=your-homey-pro-ip +HOMEY_TOKEN=your_homey_api_token_here +HOMEY_SCHEME=http +HOMEY_PORT=4859 + +# MCP server configuration (LAN mode) +HOMEY_MCP_TRANSPORT=streamable-http +HOMEY_MCP_HOST=0.0.0.0 +HOMEY_MCP_ADVERTISED_HOST= +HOMEY_MCP_PORT=8080 +HOMEY_MCP_AUTH_MODE=none +HOMEY_MCP_AUTH_TOKEN=your_mcp_auth_token_here + +# Socket.IO realtime configuration (optional) +HOMEY_REALTIME=true +EOF + fi +fi + +echo "" +echo "=== Setup complete! ===" +echo "" +echo "Next steps:" +echo "1. Edit .env with your Homey Pro credentials (HOMEY_TOKEN and HOMEY_HOST)" +echo "2. For LAN mode (recommended for Odysseus): ./run-lan.sh" +echo "3. For local stdio mode: ./run.sh" +echo "" +echo "For persistent operation, install the systemd service:" +echo " sudo cp homey-mcp.service /etc/systemd/system/" +echo " sudo systemctl daemon-reload" +echo " sudo systemctl enable --now homey-mcp@$(whoami).service" +echo "" \ No newline at end of file diff --git a/homey-mcp/src/homey_mcp/__init__.py b/homey-mcp/src/homey_mcp/__init__.py new file mode 100644 index 0000000..1e65900 --- /dev/null +++ b/homey-mcp/src/homey_mcp/__init__.py @@ -0,0 +1,3 @@ +"""homey-mcp: MCP server for a local Homey Pro instance.""" + +__version__ = "0.1.0" diff --git a/homey-mcp/src/homey_mcp/__main__.py b/homey-mcp/src/homey_mcp/__main__.py new file mode 100644 index 0000000..f5f6e40 --- /dev/null +++ b/homey-mcp/src/homey_mcp/__main__.py @@ -0,0 +1,4 @@ +from .server import main + +if __name__ == "__main__": + main() diff --git a/homey-mcp/src/homey_mcp/config.py b/homey-mcp/src/homey_mcp/config.py new file mode 100644 index 0000000..da64b11 --- /dev/null +++ b/homey-mcp/src/homey_mcp/config.py @@ -0,0 +1,42 @@ +"""Configuration loaded from environment variables / .env.""" + +from __future__ import annotations + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Settings for the homey-mcp server. + + Values are read from environment variables (prefix ``HOMEY_``) or a + local ``.env`` file. + """ + + model_config = SettingsConfigDict( + env_prefix="HOMEY_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + host: str = Field(..., description="IP or hostname of the Homey Pro.") + token: str = Field(..., description="Personal Access Token (Bearer).") + scheme: str = Field("http", description="http or https.") + port: int | None = Field( + None, + description="Optional TCP port (e.g. 4859 when Homey runs in a container).", + ) + timeout: float = Field(10.0, description="HTTP timeout in seconds.") + event_buffer: int = Field(200, description="Max realtime events to retain.") + realtime: bool = Field(True, description="Enable Socket.IO realtime events.") + + @property + def base_url(self) -> str: + if self.port is None: + return f"{self.scheme}://{self.host}" + return f"{self.scheme}://{self.host}:{self.port}" + + +def load_settings() -> Settings: + return Settings() # type: ignore[call-arg] diff --git a/homey-mcp/src/homey_mcp/homey_client.py b/homey-mcp/src/homey_mcp/homey_client.py new file mode 100644 index 0000000..72e05e4 --- /dev/null +++ b/homey-mcp/src/homey_mcp/homey_client.py @@ -0,0 +1,180 @@ +"""Async client for the Homey Pro local Web API + Socket.IO realtime events.""" + +from __future__ import annotations + +import logging +import time +from collections import deque +from typing import Any + +import httpx + +try: + import socketio # type: ignore +except ImportError: # pragma: no cover + socketio = None # type: ignore + +from .config import Settings + +log = logging.getLogger(__name__) + + +class HomeyClient: + """Thin async wrapper around Homey's local HTTP + Socket.IO APIs.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + self._http = httpx.AsyncClient( + base_url=settings.base_url, + headers={ + "Authorization": f"Bearer {settings.token}", + "Accept": "application/json", + }, + timeout=settings.timeout, + ) + self._events: deque[dict[str, Any]] = deque(maxlen=settings.event_buffer) + self._sio: Any | None = None + self._sio_connected: bool = False + + # ---- lifecycle -------------------------------------------------------- + + async def aclose(self) -> None: + try: + if self._sio is not None and self._sio_connected: + await self._sio.disconnect() + except Exception: # pragma: no cover + log.exception("Error disconnecting Socket.IO") + await self._http.aclose() + + # ---- helpers ---------------------------------------------------------- + + async def _get(self, path: str) -> Any: + r = await self._http.get(path) + r.raise_for_status() + return r.json() + + async def _post(self, path: str, json: Any | None = None) -> Any: + r = await self._http.post(path, json=json) + r.raise_for_status() + if r.content: + try: + return r.json() + except ValueError: + return None + return None + + async def _put(self, path: str, json: Any | None = None) -> Any: + r = await self._http.put(path, json=json) + r.raise_for_status() + if r.content: + try: + return r.json() + except ValueError: + return None + return None + + # ---- REST: devices / zones / flows ----------------------------------- + + async def get_system_info(self) -> dict[str, Any]: + return await self._get("/api/manager/system/info") + + async def list_devices(self) -> dict[str, Any]: + return await self._get("/api/manager/devices/device") + + async def get_device(self, device_id: str) -> dict[str, Any]: + return await self._get(f"/api/manager/devices/device/{device_id}") + + async def list_zones(self) -> dict[str, Any]: + return await self._get("/api/manager/zones/zone") + + async def set_capability( + self, device_id: str, capability_id: str, value: Any + ) -> Any: + return await self._put( + f"/api/manager/devices/device/{device_id}/capability/{capability_id}", + json={"value": value}, + ) + + async def list_flows(self) -> dict[str, Any]: + return await self._get("/api/manager/flow/flow") + + async def trigger_flow(self, flow_id: str) -> Any: + return await self._post(f"/api/manager/flow/flow/{flow_id}/trigger") + + # ---- REST: logic variables ------------------------------------------- + + async def list_variables(self) -> dict[str, Any]: + """Return all Homey Logic Variables (type, current value, etc.).""" + return await self._get("/api/manager/logic/variable") + + async def set_variable(self, variable_id: str, value: Any) -> Any: + """Set a Homey Logic Variable to *value*.""" + return await self._put( + f"/api/manager/logic/variable/{variable_id}", + json={"value": value}, + ) + + # ---- REST: notifications --------------------------------------------- + + async def send_notification(self, title: str, excerpt: str = "") -> Any: + """Push a notification to the Homey app on all registered phones.""" + return await self._post( + "/api/manager/notifications/notification", + json={ + "ownerUri": "homey:manager:notifications", + "title": title, + "excerpt": excerpt, + }, + ) + + # ---- realtime events -------------------------------------------------- + + def recent_events( + self, limit: int = 50, kind: str | None = None + ) -> list[dict[str, Any]]: + items = list(self._events) + if kind: + items = [e for e in items if e.get("kind") == kind] + return items[-limit:] + + async def start_realtime(self) -> None: + """Best-effort Socket.IO connection. Failures are logged, not raised.""" + if not self.settings.realtime: + return + if socketio is None: + log.warning("python-socketio not installed; realtime disabled") + return + if self._sio_connected: + return + + sio = socketio.AsyncClient(reconnection=True, logger=False, engineio_logger=False) + self._sio = sio + + def _record(kind: str) -> Any: + async def _handler(data: Any = None) -> None: + self._events.append( + {"ts": time.time(), "kind": kind, "data": data} + ) + + return _handler + + for kind in ( + "device.create", + "device.update", + "device.delete", + "capability", + ): + sio.on(kind, _record(kind)) + + try: + await sio.connect( + self.settings.base_url, + auth={"token": self.settings.token}, + transports=["websocket"], + wait_timeout=self.settings.timeout, + ) + self._sio_connected = True + log.info("Socket.IO connected to %s", self.settings.base_url) + except Exception as exc: # pragma: no cover + log.warning("Socket.IO connection failed: %s", exc) + self._sio_connected = False diff --git a/homey-mcp/src/homey_mcp/server.py b/homey-mcp/src/homey_mcp/server.py new file mode 100644 index 0000000..f92c7e3 --- /dev/null +++ b/homey-mcp/src/homey_mcp/server.py @@ -0,0 +1,1383 @@ +"""FastMCP server exposing the local Homey Pro instance.""" + +from __future__ import annotations + +import asyncio +import hmac +import json +import logging +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any + +import httpx +from mcp.server.fastmcp import Context, FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from pydantic import BaseModel, Field + +from .config import load_settings +from .homey_client import HomeyClient + +log = logging.getLogger("homey_mcp") + + +# ---- models -------------------------------------------------------------- + + +class DeviceSummary(BaseModel): + id: str + name: str + zone: str | None = None + class_: str | None = Field(default=None, alias="class") + available: bool | None = None + capabilities: list[str] = [] + capabilities_values: dict[str, Any] = {} + + model_config = {"populate_by_name": True} + + +class ZoneSummary(BaseModel): + id: str + name: str + parent: str | None = None + + +class FlowSummary(BaseModel): + id: str + name: str + enabled: bool | None = None + folder: str | None = None + + +class CapabilityResult(BaseModel): + device_id: str + capability: str + value: Any + + +class EventEntry(BaseModel): + ts: float + kind: str + data: Any = None + + +class PingResult(BaseModel): + ok: bool + host: str + info: dict[str, Any] | None = None + error: str | None = None + + +class VariableSummary(BaseModel): + id: str + name: str + type: str | None = None + value: Any = None + + +# ---- lifespan ------------------------------------------------------------ + + +@dataclass +class AppContext: + homey: HomeyClient + + +_shared_client: HomeyClient | None = None + + +@asynccontextmanager +async def lifespan(_server: FastMCP) -> AsyncIterator[AppContext]: + global _shared_client + settings = load_settings() + client = HomeyClient(settings) + _shared_client = client + # Start realtime connection in the background so we don't block the MCP + # initialize handshake (Socket.IO connect can take seconds or hang). + import asyncio as _asyncio + + realtime_task: _asyncio.Task | None = None + if settings.realtime: + realtime_task = _asyncio.create_task(_safe_start_realtime(client)) + try: + yield AppContext(homey=client) + finally: + _shared_client = None + if realtime_task is not None and not realtime_task.done(): + realtime_task.cancel() + await client.aclose() + + +async def _safe_start_realtime(client: HomeyClient) -> None: + try: + await client.start_realtime() + except Exception: + log.exception("Failed to start realtime listener") + +async def _rest_system(_request): + from starlette.responses import JSONResponse + + async with _resource_homey() as homey: + zones, devices = await asyncio.gather(homey.list_zones(), homey.list_devices()) + return JSONResponse({ + "ok": True, + "homey": homey.settings.host, + "zone_count": len(zones), + "device_count": len(devices), + }) + +async def _rest_zones(_request): + from starlette.responses import JSONResponse + + async with _resource_homey() as homey: + return JSONResponse(await homey.list_zones()) + +async def _rest_devices(_request): + from starlette.responses import JSONResponse + + async with _resource_homey() as homey: + return JSONResponse(await homey.list_devices()) + +async def _rest_set_capability(request): + from starlette.responses import JSONResponse + + body = await request.json() + if "value" not in body: + return JSONResponse({"error": "JSON body must contain 'value'"}, status_code=400) + device_id = request.path_params["device_id"] + capability = request.path_params["capability"] + async with _resource_homey() as homey: + device = await homey.get_device(device_id) + value = _coerce_capability_value(body["value"], _capability_type(device, capability)) + await homey.set_capability(device_id, capability, value) + return JSONResponse({ + "ok": True, + "device_id": device_id, + "capability": capability, + "value": value, + }) + +async def _rest_control_zone(request): + from starlette.responses import JSONResponse + + body = await request.json() + zone_query = str(body.get("zone", "")).strip().lower() + capability = str(body.get("capability", "")).strip() + if not zone_query or not capability or "value" not in body: + return JSONResponse( + {"error": "JSON body requires zone, capability, and value"}, status_code=400) + + async with _resource_homey() as homey: + zones = await homey.list_zones() + zone_id = None + zone_name = None + for candidate_id, zone in zones.items(): + if zone_query in str(zone.get("name", "")).lower(): + zone_id = str(candidate_id) + zone_name = str(zone.get("name", zone_query)) + break + if zone_id is None: + return JSONResponse({"error": f"Zone '{body['zone']}' not found"}, status_code=404) + + devices = await homey.list_devices() + affected: list[str] = [] + errors: list[str] = [] + for device in devices.values(): + if not isinstance(device, dict): + continue + device_zone = device.get("zone", "") + if isinstance(device_zone, dict): + device_zone = device_zone.get("id", "") + capabilities = device.get("capabilities") or [] + if str(device_zone) != zone_id or capability not in capabilities: + continue + try: + dev_value = _coerce_capability_value( + body["value"], _capability_type(device, capability) + ) + await homey.set_capability(str(device["id"]), capability, dev_value) + affected.append(str(device.get("name", device["id"]))) + except Exception as exc: + errors.append(f"{device.get('name', device.get('id'))}: {exc}") + + return JSONResponse({ + "ok": not errors, + "zone": zone_name, + "capability": capability, + "value": body["value"], + "affected": affected, + "errors": errors, + }) + +async def _rest_openapi(request): + from starlette.responses import JSONResponse + + value_schema = {"oneOf": [ + {"type": "boolean"}, {"type": "number"}, {"type": "string"} + ]} + return JSONResponse({ + "openapi": "3.1.0", + "info": {"title": "Homey API", "version": "1.0.0"}, + "servers": [{"url": str(request.base_url).rstrip("/")}], + "components": {"securitySchemes": {"BearerAuth": { + "type": "http", "scheme": "bearer" + }}}, + "security": [{"BearerAuth": []}], + "paths": { + "/api/system": {"get": { + "operationId": "get_homey_system", + "summary": "Check Homey connectivity and counts", + "responses": {"200": {"description": "Homey status"}}, + }}, + "/api/zones": {"get": { + "operationId": "list_homey_zones", + "summary": "List Homey rooms/zones", + "responses": {"200": {"description": "Zones"}}, + }}, + "/api/devices": {"get": { + "operationId": "list_homey_devices", + "summary": "List devices with IDs and current state", + "responses": {"200": {"description": "Devices"}}, + }}, + "/api/devices/{device_id}/capabilities/{capability}": {"put": { + "operationId": "set_homey_device_capability", + "summary": "Set a capability on one Homey device", + "parameters": [ + {"name": "device_id", "in": "path", "required": True, + "schema": {"type": "string"}}, + {"name": "capability", "in": "path", "required": True, + "schema": {"type": "string"}}, + ], + "requestBody": {"required": True, "content": { + "application/json": {"schema": {"type": "object", + "required": ["value"], "properties": {"value": value_schema}}} + }}, + "responses": {"200": {"description": "Capability updated"}}, + }}, + "/api/zones/control": {"post": { + "operationId": "control_homey_zone", + "summary": "Set a capability on all supporting devices in a room", + "requestBody": {"required": True, "content": { + "application/json": {"schema": {"type": "object", + "required": ["zone", "capability", "value"], "properties": { + "zone": {"type": "string"}, + "capability": {"type": "string"}, + "value": value_schema, + }}} + }}, + "responses": {"200": {"description": "Zone controlled"}}, + }}, + }, + }) + + +# For LAN transports (sse / streamable-http), disable DNS rebinding protection +# and allow any Host header, since this is a trusted-LAN MCP server and clients +# will connect with their local IP address. +mcp = FastMCP( + "homey-mcp", + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=False, + allowed_hosts=[], + allowed_origins=[], + ), + instructions=( + "Tools for inspecting and controlling a local Homey Pro smart-home hub.\n\n" + "IMPORTANT WORKFLOW: Device IDs in Homey are UUIDs and CANNOT be guessed " + "from names. Before calling set_capability, set_light, get_device, or " + "get_capability, ALWAYS use `find_devices` (name search) or `list_devices` " + "to obtain the real device_id. Never invent IDs.\n\n" + "AVAILABLE TOOLS SUMMARY:\n" + " Connectivity: ping\n" + " Devices: list_devices, find_devices, get_device\n" + " Zones: list_zones\n" + " Status views: get_zone_status, get_house_status, get_temperatures\n" + " Capabilities: get_capability, set_capability, set_light, set_capabilities_bulk\n" + " Zone control: control_zone, control_all\n" + " Flows: list_flows, find_flows, trigger_flow\n" + " Variables: list_variables, set_variable\n" + " Notifications: send_notification\n" + " Events: list_recent_events\n\n" + "COMMON WORKFLOWS:\n" + " 'What's on in the kitchen?' → get_zone_status(zone='Kitchen')\n" + " 'Turn off all lights' → control_all(capability='onoff', value=False, device_class='light')\n" + " 'Set living-room lights warm' → find_devices('living room light') → set_light(...)\n" + " 'Trigger good-night routine' → find_flows('good night') → trigger_flow(id)\n" + " 'What's the temperature?' → get_temperatures()\n" + " 'Set variable X' → list_variables() → set_variable(id, value)\n\n" + "Common capabilities: `onoff` (bool), `dim` (0..1 float), " + "`target_temperature` (number °C), `measure_temperature` (read-only °C), " + "`light_hue` (0..1), `light_saturation` (0..1), `light_temperature` (0..1 warm→cold)." + ), + lifespan=lifespan, +) + + +# ---- helpers ------------------------------------------------------------- + + +def _homey(ctx: Context) -> HomeyClient: + app: AppContext = ctx.request_context.lifespan_context # type: ignore[assignment] + return app.homey + + +def _capability_type(device: dict[str, Any], capability: str) -> str | None: + """Look up a capability's declared type (e.g. 'boolean', 'number') from + a device object's capabilitiesObj, if present.""" + obj = device.get("capabilitiesObj") or {} + cap = obj.get(capability) if isinstance(obj, dict) else None + return cap.get("type") if isinstance(cap, dict) else None + + +def _coerce_capability_value(value: Any, cap_type: str | None) -> Any: + """Coerce a loosely-typed tool argument to what Homey's local API expects. + + MCP clients (and the LLMs driving them) often send booleans as strings + ("true"/"off") or numbers (0/1) instead of a native JSON boolean, which + Homey's strict local API rejects with 400 Bad Request. + """ + if cap_type == "boolean": + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in ("true", "on", "yes", "1"): + return True + if v in ("false", "off", "no", "0"): + return False + return value + if cap_type == "number": + if isinstance(value, bool): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value + return value + if cap_type == "string" and not isinstance(value, str): + return str(value) + return value + + +def _device_to_summary(d: dict[str, Any]) -> DeviceSummary: + caps = d.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + cap_values: dict[str, Any] = {} + obj = d.get("capabilitiesObj") or {} + if isinstance(obj, dict): + for cap_id, cap in obj.items(): + if isinstance(cap, dict) and "value" in cap: + cap_values[cap_id] = cap.get("value") + return DeviceSummary( + id=str(d.get("id", "")), + name=str(d.get("name", "")), + zone=d.get("zone"), + **{"class": d.get("class")}, + available=d.get("available"), + capabilities=[str(c) for c in caps], + capabilities_values=cap_values, + ) + + +# ---- tools --------------------------------------------------------------- + + +@mcp.tool() +async def ping(ctx: Context) -> PingResult: + """Verify connectivity to the Homey Pro and return inventory counts.""" + homey = _homey(ctx) + try: + devices, zones = await asyncio.gather(homey.list_devices(), homey.list_zones()) + return PingResult( + ok=True, + host=homey.settings.host, + info={"devices": len(devices), "zones": len(zones)}, + ) + except Exception as exc: + return PingResult(ok=False, host=homey.settings.host, error=str(exc)) + + +@mcp.tool() +async def list_devices( + ctx: Context, zone_id: str | None = None +) -> list[DeviceSummary]: + """List ALL devices on Homey with their real UUIDs, names, zones, and + current capability values. Use this (or `find_devices`) to obtain the + real `device_id` before calling `set_capability` / `get_device`. + """ + homey = _homey(ctx) + raw = await homey.list_devices() + items = raw.values() if isinstance(raw, dict) else raw + out = [_device_to_summary(d) for d in items if isinstance(d, dict)] + if zone_id: + out = [d for d in out if d.zone == zone_id] + return out + + +@mcp.tool() +async def find_devices( + ctx: Context, query: str, zone: str | None = None +) -> list[DeviceSummary]: + """Search devices by a free-text query (case-insensitive substring match + against the device name and zone name). Returns matching devices with + their real `id` (UUID). Use this BEFORE `set_capability` when the user + refers to a device by a human name like 'Wohnzimmerlicht' or 'kitchen + lamp'. + + Args: + query: substring to match against device name (and zone name). + zone: optional substring to match against zone name to narrow down. + """ + homey = _homey(ctx) + raw_devices = await homey.list_devices() + raw_zones = await homey.list_zones() + zones_by_id: dict[str, str] = {} + if isinstance(raw_zones, dict): + for zid, z in raw_zones.items(): + if isinstance(z, dict): + zones_by_id[str(zid)] = str(z.get("name", "")) + + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + # Split the query into individual words so multi-word queries like + # "esszimmer licht" still match a device whose zone is "Esszimmer". + q_words = [w for w in (query or "").strip().lower().split() if w] + zq = (zone or "").strip().lower() or None + + def _zone_name_for(d: dict) -> str: + """Return the zone display-name for a device, handling both UUID and + embedded-object formats that the Homey API may return.""" + zone_field = d.get("zone", "") + if isinstance(zone_field, dict): + # Homey sometimes returns zone as {"id": ..., "name": ...} + return str(zone_field.get("name", "")).lower() + # Normal case: zone is a UUID string; look it up in zones_by_id. + return zones_by_id.get(str(zone_field), "").lower() + + matches: list[DeviceSummary] = [] + for d in items: + if not isinstance(d, dict): + continue + name = str(d.get("name", "")).lower() + zone_name = _zone_name_for(d) + # Match if ANY query word appears in the device name or zone name. + if q_words and not any(w in name or w in zone_name for w in q_words): + continue + if zq and zq not in zone_name: + continue + matches.append(_device_to_summary(d)) + return matches + + +@mcp.tool() +async def get_device(ctx: Context, device_id: str) -> dict[str, Any]: + """Get the full device object (including all capability values).""" + homey = _homey(ctx) + return await homey.get_device(device_id) + + +@mcp.tool() +async def list_zones(ctx: Context) -> list[ZoneSummary]: + """List all zones (rooms / areas).""" + homey = _homey(ctx) + raw = await homey.list_zones() + items = raw.values() if isinstance(raw, dict) else raw + return [ + ZoneSummary( + id=str(z.get("id", "")), + name=str(z.get("name", "")), + parent=z.get("parent"), + ) + for z in items + if isinstance(z, dict) + ] + + +@mcp.tool() +async def set_capability( + ctx: Context, + device_id: str, + capability: str, + value: bool | int | float | str, +) -> CapabilityResult: + """Set a capability value on a device. + + `device_id` MUST be the real Homey UUID obtained from `find_devices` or + `list_devices` — it is NOT the device name. If you do not yet know it, + call `find_devices` first. + + Common capabilities: `onoff` (bool), `dim` (0..1 float), + `target_temperature` (number, °C), `light_hue` (0..1 float). + """ + homey = _homey(ctx) + # Verify the device exists first so we can return a clear error instead + # of a generic HTTP failure (which some LLMs misinterpret as success). + try: + device = await homey.get_device(device_id) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + raise ValueError( + f"No device with id {device_id!r}. Call `find_devices` to look " + "up the real UUID by name." + ) from exc + raise + caps = device.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + if capability not in caps: + raise ValueError( + f"Device {device.get('name')!r} ({device_id}) does not support " + f"capability {capability!r}. Available: {sorted(caps)}" + ) + value = _coerce_capability_value(value, _capability_type(device, capability)) + await homey.set_capability(device_id, capability, value) + return CapabilityResult(device_id=device_id, capability=capability, value=value) + + +@mcp.tool() +async def control_zone( + ctx: Context, + zone: str, + capability: str, + value: bool | int | float | str, +) -> dict[str, Any]: + """Set a capability on ALL devices in a named zone in one call. + + Use this instead of calling `find_devices` + `set_capability` N times + when the user wants to control an entire room (e.g. 'turn off Esszimmer', + 'dim Wohnzimmer to 50%', 'Küche Licht an'). + + Zone matching is case-insensitive substring: 'wohn' matches 'Wohnzimmer'. + If multiple zones match, the FIRST match is used — use a more specific + substring (e.g. 'Wohnzimmer' not 'zimmer') to avoid ambiguity. + + Devices that don't support the capability are silently skipped (reported + in the `skipped` list). Check `affected` > 0 to confirm something changed. + + Args: + zone: substring to match against zone name (case-insensitive). + Prefer the exact zone name to avoid multi-match ambiguity. + capability: capability to set on every matched device. + Common: 'onoff' (bool), 'dim' (0..1), 'light_hue' (0..1), + 'light_saturation' (0..1), 'light_temperature' (0..1, warm→cold), + 'target_temperature' (°C number). + value: value to set. + + Returns a summary with `affected` (number of devices updated), + `skipped` (devices without the capability), and `errors`. + """ + homey = _homey(ctx) + + # --- Step 1: resolve zone name → UUID --- + try: + raw_zones = await homey.list_zones() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_zones): {type(exc).__name__}" + ) from None + zone_id: str | None = None + zone_display = zone + zq = zone.strip().lower() + if isinstance(raw_zones, dict): + for zid, z in raw_zones.items(): + if isinstance(z, dict) and zq in str(z.get("name", "")).lower(): + zone_id = str(zid) + zone_display = str(z.get("name", zone)) + break + + if zone_id is None: + raise ValueError( + f"Zone '{zone}' not found. Call list_zones to see available zones." + ) + + # --- Step 2: collect devices whose zone field matches zone_id --- + try: + raw_devices = await homey.list_devices() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_devices): {type(exc).__name__}" + ) from None + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + + zone_devices: list[dict] = [] + for d in items: + if not isinstance(d, dict): + continue + zone_field = d.get("zone", "") + if isinstance(zone_field, dict): + # Homey sometimes embeds zone as {"id": "...", "name": "..."} + match = (str(zone_field.get("id", "")) == zone_id + or zq in str(zone_field.get("name", "")).lower()) + else: + match = str(zone_field) == zone_id + if match: + zone_devices.append(d) + + if not zone_devices: + raise ValueError( + f"No devices found in zone '{zone_display}' (id={zone_id}). " + "The zone exists but contains no devices — check list_devices." + ) + + # --- Step 3: set capability on each device (sequential, full error capture) --- + affected: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + + for d in zone_devices: + dev_id = str(d.get("id") or "") + dev_name = str(d.get("name") or dev_id or "?") + caps = d.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + if capability not in caps: + skipped.append(dev_name) + continue + try: + dev_value = _coerce_capability_value(value, _capability_type(d, capability)) + await homey.set_capability(dev_id, capability, dev_value) + affected.append(dev_name) + except Exception as exc: + errors.append(f"{dev_name}: {exc}") + + return { + "zone": zone_display, + "capability": capability, + "value": value, + "affected": len(affected), + "devices": affected, + "skipped": skipped, + "errors": errors, + } + + +@mcp.tool() +async def list_flows(ctx: Context) -> list[FlowSummary]: + """List all Homey flows.""" + homey = _homey(ctx) + raw = await homey.list_flows() + items = raw.values() if isinstance(raw, dict) else raw + return [ + FlowSummary( + id=str(f.get("id", "")), + name=str(f.get("name", "")), + enabled=f.get("enabled"), + folder=f.get("folder"), + ) + for f in items + if isinstance(f, dict) + ] + + +@mcp.tool() +async def trigger_flow(ctx: Context, flow_id: str) -> dict[str, Any]: + """Trigger (start) a Homey flow by id.""" + homey = _homey(ctx) + result = await homey.trigger_flow(flow_id) + return {"flow_id": flow_id, "result": result, "ok": True} + + +@mcp.tool() +async def list_recent_events( + ctx: Context, limit: int = 50, kind: str | None = None +) -> list[EventEntry]: + """Return recent realtime events captured from Homey's Socket.IO stream. + + ``kind`` filters to one of: ``device.create``, ``device.update``, + ``device.delete``, ``capability``. + """ + homey = _homey(ctx) + return [EventEntry(**e) for e in homey.recent_events(limit=limit, kind=kind)] + + +# ---- extended tools ------------------------------------------------------ + + +@mcp.tool() +async def get_zone_status(ctx: Context, zone: str) -> dict[str, Any]: + """Get the current state of ALL devices inside a named zone. + + Returns each device's name, class, availability, and all current + capability values (on/off state, brightness, temperature, etc.). + Perfect for answering "what's on in the kitchen?" or "is the bedroom + light on?". + + Args: + zone: substring to match against zone name (case-insensitive). + """ + homey = _homey(ctx) + try: + raw_zones = await homey.list_zones() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_zones): {type(exc).__name__}" + ) from None + zone_id: str | None = None + zone_display = zone + zq = zone.strip().lower() + if isinstance(raw_zones, dict): + for zid, z in raw_zones.items(): + if isinstance(z, dict) and zq in str(z.get("name", "")).lower(): + zone_id = str(zid) + zone_display = str(z.get("name", zone)) + break + if zone_id is None: + raise ValueError(f"Zone '{zone}' not found. Call list_zones for available zones.") + + try: + raw_devices = await homey.list_devices() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_devices): {type(exc).__name__}" + ) from None + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + devices: list[dict[str, Any]] = [] + for d in items: + if not isinstance(d, dict): + continue + zone_field = d.get("zone", "") + if isinstance(zone_field, dict): + match = str(zone_field.get("id", "")) == zone_id + else: + match = str(zone_field) == zone_id + if not match: + continue + s = _device_to_summary(d) + devices.append({ + "name": s.name, + "id": s.id, + "class": s.class_, + "available": s.available, + "capabilities": s.capabilities, + "state": s.capabilities_values, + }) + return { + "zone": zone_display, + "zone_id": zone_id, + "device_count": len(devices), + "devices": devices, + } + + +@mcp.tool() +async def get_house_status( + ctx: Context, class_filter: str | None = None +) -> dict[str, Any]: + """Get a full home overview: for each zone, list all devices and their + current state (capabilities + values). + + Optionally filter by device class to focus on a single type. + + Args: + class_filter: optional device class, e.g. 'light', 'thermostat', + 'socket', 'sensor', 'speaker', 'fan', 'lock'. + Omit to get everything. + """ + homey = _homey(ctx) + try: + raw_devices = await homey.list_devices() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_devices): {type(exc).__name__}" + ) from None + try: + raw_zones = await homey.list_zones() + except Exception as exc: + raise RuntimeError( + f"Could not reach Homey (list_zones): {type(exc).__name__}" + ) from None + zones_by_id: dict[str, str] = {} + if isinstance(raw_zones, dict): + for zid, z in raw_zones.items(): + if isinstance(z, dict): + zones_by_id[str(zid)] = str(z.get("name", "?")) + + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + by_zone: dict[str, list[dict[str, Any]]] = {} + for d in items: + if not isinstance(d, dict): + continue + if class_filter and d.get("class") != class_filter: + continue + zone_field = d.get("zone", "") + if isinstance(zone_field, dict): + zone_name = str(zone_field.get("name", "?")) + else: + zone_name = zones_by_id.get(str(zone_field), "?") + s = _device_to_summary(d) + by_zone.setdefault(zone_name, []).append({ + "name": s.name, + "id": s.id, + "class": s.class_, + "available": s.available, + "state": s.capabilities_values, + }) + return { + "filter": class_filter or "all", + "zones": [ + {"zone": zn, "devices": devs} + for zn, devs in sorted(by_zone.items()) + ], + "total_devices": sum(len(v) for v in by_zone.values()), + } + + +@mcp.tool() +async def get_temperatures(ctx: Context) -> list[dict[str, Any]]: + """Get every temperature-related reading across the entire home. + + Returns `measure_temperature` (current reading) and `target_temperature` + (thermostat setpoint) for all devices that expose them, grouped by zone. + Perfect for 'what's the temperature in each room?' or checking thermostat + targets. + """ + homey = _homey(ctx) + try: + raw_devices = await homey.list_devices() + except Exception as exc: + return [{"error": f"Could not reach Homey (list_devices): {exc or type(exc).__name__}"}] + try: + raw_zones = await homey.list_zones() + except Exception as exc: + return [{"error": f"Could not reach Homey (list_zones): {exc or type(exc).__name__}"}] + zones_by_id: dict[str, str] = {} + if isinstance(raw_zones, dict): + for zid, z in raw_zones.items(): + if isinstance(z, dict): + zones_by_id[str(zid)] = str(z.get("name", "?")) + + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + results: list[dict[str, Any]] = [] + for d in items: + if not isinstance(d, dict): + continue + cap_obj = d.get("capabilitiesObj") or {} + row: dict[str, Any] = {} + for cap_name in ("measure_temperature", "target_temperature"): + if cap_name in cap_obj: + val = cap_obj[cap_name] + row[cap_name] = val.get("value") if isinstance(val, dict) else val + if not row: + continue + zone_field = d.get("zone", "") + if isinstance(zone_field, dict): + zone_name = str(zone_field.get("name", "?")) + else: + zone_name = zones_by_id.get(str(zone_field), "?") + results.append({ + "name": d.get("name"), + "zone": zone_name, + "class": d.get("class"), + **row, + }) + results.sort(key=lambda x: (x.get("zone") or "", x.get("name") or "")) + return results + + +@mcp.tool() +async def find_flows(ctx: Context, query: str) -> list[FlowSummary]: + """Search flows by name (case-insensitive substring match). + + Use this to discover the real flow UUID before calling `trigger_flow`. + Much faster than calling `list_flows` and scanning manually. + + Args: + query: substring to match in the flow name. + """ + homey = _homey(ctx) + raw = await homey.list_flows() + items = raw.values() if isinstance(raw, dict) else raw + q = query.strip().lower() + return [ + FlowSummary( + id=str(f.get("id", "")), + name=str(f.get("name", "")), + enabled=f.get("enabled"), + folder=f.get("folder"), + ) + for f in items + if isinstance(f, dict) and q in str(f.get("name", "")).lower() + ] + + +@mcp.tool() +async def get_capability( + ctx: Context, device_id: str, capability: str +) -> dict[str, Any]: + """Read the current value of a single capability from a device. + + Use this to check a specific property without fetching the full device + object. E.g. check if a switch is on, read the current dimmer level, + or query a temperature sensor. + + Args: + device_id: Homey device UUID (from find_devices / list_devices). + capability: capability id, e.g. 'onoff', 'dim', 'measure_temperature', + 'target_temperature', 'measure_humidity'. + """ + homey = _homey(ctx) + device = await homey.get_device(device_id) + cap_obj = (device.get("capabilitiesObj") or {}).get(capability) + if cap_obj is None: + caps = device.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + raise ValueError( + f"Device {device.get('name')!r} has no capability {capability!r}. " + f"Available: {sorted(caps)}" + ) + return { + "device_id": device_id, + "device_name": device.get("name"), + "capability": capability, + "value": cap_obj.get("value") if isinstance(cap_obj, dict) else cap_obj, + "lastChanged": cap_obj.get("lastChanged") if isinstance(cap_obj, dict) else None, + } + + +@mcp.tool() +async def set_light( + ctx: Context, + device_id: str, + on: bool | None = None, + dim: float | None = None, + hue: float | None = None, + saturation: float | None = None, + temperature: float | None = None, +) -> dict[str, Any]: + """Set one or more light properties in a single call. + + Replaces multiple separate `set_capability` calls for lights. All + parameters are optional; only the ones you provide are applied, in the + correct order (onoff → dim → colour properties). + + Args: + device_id: Homey device UUID. + on: True to turn on, False to turn off. + dim: Brightness 0.0 (off) – 1.0 (full brightness). + hue: Colour hue 0.0–1.0 (0/1=red, 0.33=green, 0.67=blue). + saturation: Colour saturation 0.0 (white) – 1.0 (pure colour). + temperature: White colour temperature 0.0 (warm/yellow) – 1.0 (cold/blue). + """ + homey = _homey(ctx) + device = await homey.get_device(device_id) + caps = device.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + + to_set: list[tuple[str, Any]] = [] + if on is not None and "onoff" in caps: + to_set.append(("onoff", on)) + if dim is not None and "dim" in caps: + to_set.append(("dim", max(0.0, min(1.0, float(dim))))) + if hue is not None and "light_hue" in caps: + to_set.append(("light_hue", max(0.0, min(1.0, float(hue))))) + if saturation is not None and "light_saturation" in caps: + to_set.append(("light_saturation", max(0.0, min(1.0, float(saturation))))) + if temperature is not None and "light_temperature" in caps: + to_set.append(("light_temperature", max(0.0, min(1.0, float(temperature))))) + + if not to_set: + raise ValueError( + f"Nothing to set. Device {device.get('name')!r} supports: {sorted(caps)}" + ) + + applied: list[dict[str, Any]] = [] + errors: list[str] = [] + for cap, val in to_set: + try: + await homey.set_capability(device_id, cap, val) + applied.append({"capability": cap, "value": val}) + except Exception as exc: + errors.append(f"{cap}: {exc}") + return { + "device_id": device_id, + "device_name": device.get("name"), + "applied": applied, + "errors": errors, + } + + +@mcp.tool() +async def set_capabilities_bulk( + ctx: Context, + device_id: str, + capabilities: dict[str, Any], +) -> dict[str, Any]: + """Set multiple capabilities on one device in a single tool call. + + Useful for scene setup (e.g. turn on, set brightness AND colour in one + go) without multiple round-trips. Capabilities are applied sequentially + in the order provided. + + Args: + device_id: Homey device UUID. + capabilities: Mapping of capability id → value, e.g. + {"onoff": true, "dim": 0.5, "light_temperature": 0.2} + """ + homey = _homey(ctx) + device = await homey.get_device(device_id) + applied: list[dict[str, Any]] = [] + errors: list[str] = [] + for cap, val in capabilities.items(): + try: + val = _coerce_capability_value(val, _capability_type(device, cap)) + await homey.set_capability(device_id, cap, val) + applied.append({"capability": cap, "value": val}) + except Exception as exc: + errors.append(f"{cap}: {exc}") + return {"device_id": device_id, "applied": applied, "errors": errors} + + +@mcp.tool() +async def control_all( + ctx: Context, + capability: str, + value: Any, + device_class: str | None = None, +) -> dict[str, Any]: + """Set a capability on ALL devices in the home (optionally filtered by class). + + Use this for home-wide commands: + 'Alle Lichter aus' → control_all(capability='onoff', value=False, device_class='light') + 'Alles ausschalten' → control_all(capability='onoff', value=False) + 'Alle Steckdosen aus' → control_all(capability='onoff', value=False, device_class='socket') + + ALWAYS pass device_class='light' for light-only commands — without it, ALL + devices that support `onoff` (sockets, fans, etc.) are also switched. + + CAUTION: Affects every matching device in the entire home. + Setting a capability other than 'onoff' or 'dim' without a device_class is + refused as a safety measure. + + Args: + capability: Capability id to set on every matching device. + value: Value to set (bool for onoff, float for dim, etc.). + device_class: Optional class filter. Common values: 'light', 'socket', + 'thermostat', 'fan', 'lock', 'speaker'. + Omit only for truly all-device commands like 'turn off everything'. + """ + if not device_class and capability not in ("onoff", "dim"): + return { + "error": ( + f"Refusing to set '{capability}' on ALL devices without a class filter. " + "Specify device_class (e.g. 'light', 'socket') to narrow the scope." + ) + } + homey = _homey(ctx) + raw_devices = await homey.list_devices() + items = raw_devices.values() if isinstance(raw_devices, dict) else raw_devices + affected: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + for d in items: + if not isinstance(d, dict): + continue + if device_class and d.get("class") != device_class: + continue + caps = d.get("capabilities") or [] + if isinstance(caps, dict): + caps = list(caps.keys()) + dev_id = str(d.get("id") or "") + dev_name = str(d.get("name") or dev_id or "?") + if capability not in caps: + skipped.append(dev_name) + continue + try: + dev_value = _coerce_capability_value(value, _capability_type(d, capability)) + await homey.set_capability(dev_id, capability, dev_value) + affected.append(dev_name) + except Exception as exc: + errors.append(f"{dev_name}: {exc}") + return { + "capability": capability, + "value": value, + "device_class": device_class or "all", + "affected_count": len(affected), + "devices": affected, + "skipped_no_capability": len(skipped), + "errors": errors, + } + + +@mcp.tool() +async def list_variables(ctx: Context) -> list[VariableSummary]: + """List all Homey Logic Variables (boolean, number, string, enum). + + Logic Variables hold state that can be read/written and used in flow + conditions and actions. Call this to see current values and find the + variable's UUID before calling `set_variable`. + """ + homey = _homey(ctx) + raw = await homey.list_variables() + items = raw.values() if isinstance(raw, dict) else raw + return [ + VariableSummary( + id=str(v.get("id", "")), + name=str(v.get("name", "")), + type=v.get("type"), + value=v.get("value"), + ) + for v in items + if isinstance(v, dict) + ] + + +@mcp.tool() +async def set_variable( + ctx: Context, + variable_id: str, + value: bool | int | float | str, +) -> dict[str, Any]: + """Set a Homey Logic Variable by its UUID. + + Logic Variables are used in flow conditions and actions. Call + `list_variables` first to find the variable's UUID and current type, + then pass the correct value type (bool for boolean variables, number for + numeric ones, string for text/enum ones). + + Args: + variable_id: UUID of the variable (from list_variables). + value: New value — must match the variable's type. + """ + homey = _homey(ctx) + await homey.set_variable(variable_id, value) + return {"variable_id": variable_id, "value": value, "ok": True} + + +@mcp.tool() +async def send_notification( + ctx: Context, title: str, excerpt: str = "" +) -> dict[str, Any]: + """Push a notification to the Homey app on all registered phones. + + Use this to alert the user of an important action you just completed, + a warning, or any status update that deserves attention. + + Args: + title: Short notification headline (keep under 80 chars). + excerpt: Optional longer message body shown below the title. + """ + homey = _homey(ctx) + try: + result = await homey.send_notification(title, excerpt) + return {"ok": True, "title": title, "result": result} + except Exception as exc: + return {"ok": False, "title": title, "error": str(exc)} + + + + + +@mcp.resource("homey://devices") +async def resource_devices() -> str: + """Snapshot of all devices as JSON.""" + # Resources don't receive Context; build a transient call via a fresh + # client would be wasteful — instead reach into the running lifespan via + # the FastMCP context manager if available. + async with _resource_homey() as homey: + raw = await homey.list_devices() + items = list(raw.values()) if isinstance(raw, dict) else raw + return json.dumps( + [_device_to_summary(d).model_dump(by_alias=True) for d in items if isinstance(d, dict)], + indent=2, + ) + + +@mcp.resource("homey://zones") +async def resource_zones() -> str: + """Snapshot of all zones as JSON.""" + async with _resource_homey() as homey: + raw = await homey.list_zones() + return json.dumps(raw, indent=2) + + +@mcp.resource("homey://events/recent") +async def resource_events() -> str: + """Recent realtime events as JSON (most recent last).""" + async with _resource_homey() as homey: + return json.dumps(homey.recent_events(limit=homey.settings.event_buffer), indent=2) + + +@asynccontextmanager +async def _resource_homey() -> AsyncIterator[HomeyClient]: + """Provide the lifespan-managed HomeyClient to resource handlers. + + FastMCP resource callbacks don't receive a ``Context``; we look up the + client stored at lifespan start. If the server is somehow being invoked + without a running lifespan (e.g. tooling import), fall back to a + short-lived client. + """ + if _shared_client is not None: + yield _shared_client + return + settings = load_settings() + tmp = HomeyClient(settings) + try: + yield tmp + finally: + await tmp.aclose() + + +# ---- entry point --------------------------------------------------------- + + +class _BearerAuthMiddleware: + """Protect REST routes and optionally MCP traffic with a static token.""" + + def __init__(self, app, token: str, protect_mcp: bool): + self.app = app + self.token = token + self.protect_mcp = protect_mcp + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + path = scope.get("path", "") + method = scope.get("method", "") + headers = dict(scope.get("headers", [])) + accepts_events = b"text/event-stream" in headers.get(b"accept", b"") + is_root_mcp = path == "/" and (method == "POST" or accepts_events) + requires_auth = path.startswith("/api/") or ( + self.protect_mcp and (path == "/mcp" or is_root_mcp) + ) + if not requires_auth: + await self.app(scope, receive, send) + return + from starlette.requests import Request + from starlette.responses import JSONResponse + + request = Request(scope) + auth = request.headers.get("authorization", "") + presented = auth.removeprefix("Bearer ").strip() if auth.startswith("Bearer ") else "" + if not presented or not hmac.compare_digest(presented, self.token): + response = JSONResponse( + {"error": "unauthorized"}, + status_code=401, + headers={"WWW-Authenticate": 'Bearer realm="homey-mcp"'}, + ) + await response(scope, receive, send) + return + await self.app(scope, receive, send) + + +class _RootMcpAliasMiddleware: + """Map MCP traffic sent to `/` onto FastMCP's `/mcp` endpoint.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and scope.get("path") == "/": + headers = dict(scope.get("headers", [])) + accepts_events = b"text/event-stream" in headers.get(b"accept", b"") + if scope.get("method") == "POST" or accepts_events: + scope = {**scope, "path": "/mcp", "raw_path": b"/mcp"} + await self.app(scope, receive, send) + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + # HOMEY_MCP_* transport vars aren't modeled in Settings (pydantic-settings' + # own .env loading doesn't populate os.environ) — load .env explicitly so + # they can be set there too, not just as real environment variables. + from dotenv import load_dotenv + load_dotenv() + + transport = os.environ.get("HOMEY_MCP_TRANSPORT", "stdio").strip().lower() + if transport not in ("stdio", "sse", "streamable-http"): + log.warning(f"Unknown HOMEY_MCP_TRANSPORT={transport!r} — falling back to stdio") + transport = "stdio" + + if transport == "stdio": + mcp.run() # unchanged local behaviour for Javis's own MCPHost (stdio) + return + + # LAN transport (sse / streamable-http) — e.g. for a remote AI helper + # such as Odysseus. Bound to 0.0.0.0 by default so it's reachable from + # other machines; restrict with a firewall rule and/or HOMEY_MCP_AUTH_TOKEN. + host = os.environ.get("HOMEY_MCP_HOST", "0.0.0.0") + advertised_host = os.environ.get("HOMEY_MCP_ADVERTISED_HOST", host) + port = int(os.environ.get("HOMEY_MCP_PORT", "8080")) + token = os.environ.get("HOMEY_MCP_AUTH_TOKEN", "").strip() + auth_mode = os.environ.get("HOMEY_MCP_AUTH_MODE", "none").strip().lower() + if auth_mode not in ("none", "bearer"): + log.warning(f"Unknown HOMEY_MCP_AUTH_MODE={auth_mode!r} — using 'none'") + auth_mode = "none" + if auth_mode == "bearer" and not token: + raise RuntimeError("HOMEY_MCP_AUTH_MODE=bearer requires HOMEY_MCP_AUTH_TOKEN") + + # Use advertised_host for MCP transport validation (Host header check). + # Bind uvicorn to host (which can be 0.0.0.0 for all interfaces). + mcp.settings.host = advertised_host + mcp.settings.port = port + app = mcp.streamable_http_app() if transport == "streamable-http" else mcp.sse_app() + + from starlette.responses import JSONResponse + from starlette.routing import Route + + async def _discovery(_request): + endpoint = mcp.settings.streamable_http_path if transport == "streamable-http" else mcp.settings.sse_path + return JSONResponse({ + "service": "homey-mcp", + "status": "ok", + "transport": transport, + "mcp_endpoint": endpoint, + "openapi": "/openapi.json", + "mcp_authentication": auth_mode, + "rest_authentication": "bearer" if token else "none", + }) + + app.routes[0:0] = [ + Route("/", _discovery, methods=["GET"]), + Route("/openapi.json", _rest_openapi, methods=["GET"]), + Route("/api/system", _rest_system, methods=["GET"]), + Route("/api/zones", _rest_zones, methods=["GET"]), + Route("/api/devices", _rest_devices, methods=["GET"]), + Route( + "/api/devices/{device_id}/capabilities/{capability}", + _rest_set_capability, + methods=["PUT"], + ), + Route("/api/zones/control", _rest_control_zone, methods=["POST"]), + ] + if transport == "streamable-http": + app = _RootMcpAliasMiddleware(app) + + if token: + app = _BearerAuthMiddleware(app, token, protect_mcp=auth_mode == "bearer") + if auth_mode == "bearer": + log.info( + f"homey-mcp ({transport}) bound to http://{host}:{port} " + f"(advertised as http://{advertised_host}:{port}) — " + "MCP + REST Bearer auth enabled" + ) + elif token: + log.warning( + f"homey-mcp ({transport}) bound to http://{host}:{port} " + f"(advertised as http://{advertised_host}:{port}) — " + "MCP is unauthenticated for client compatibility; REST API requires Bearer auth" + ) + else: + log.warning( + f"homey-mcp ({transport}) bound to http://{host}:{port} " + f"(advertised as http://{advertised_host}:{port}) WITHOUT authentication — " + "anyone on the LAN can control your smart home. Set HOMEY_MCP_AUTH_TOKEN to secure it." + ) + + import uvicorn + uvicorn.run(app, host=host, port=port, log_level="info") + + +if __name__ == "__main__": + main() + diff --git a/javis-slave-mcp/README.md b/javis-slave-mcp/README.md new file mode 100644 index 0000000..a46d523 --- /dev/null +++ b/javis-slave-mcp/README.md @@ -0,0 +1,150 @@ +# Javis Slave MCP + +MCP (Model Context Protocol) server for controlling local Javis Slave devices. +Slaves can run on Windows tablets or Linux boards (Raspberry Pi, see +[javis-slave/RASPBERRY_PI.md](../javis-slave/RASPBERRY_PI.md)); the control +endpoints are identical on both. + +## Features + +- **Status Monitoring**: Check device status, audio state, mic level, muted state +- **Device Info**: Get device configuration (sample rate, audio devices, wake word model) +- **Network Discovery**: Query the master for the IP of any slave, or list all connected slaves +- **Microphone Control**: Mute/unmute/toggle microphone on slave devices +- **Audio Engine Recovery**: Restart audio engine (stop + start) when needed +- **Volume Control**: Get/set/increase/decrease master volume, mute/unmute output +- **Brightness Control**: Get/set/increase/decrease display brightness +- **Power Control**: Sleep / shutdown / restart / cancel shutdown + +## Configuration + +### Environment Variables + +```bash +# Comma-separated list of device_id=host:port +export JAVIS_SLAVE_DEVICES="device1=192.168.1.100:8766,device2=192.168.1.101:8766" +``` + +### Config File + +Create `~/.javis/slave_mcp_config.json`: + +```json +{ + "slaves": { + "bedroom": { + "host": "192.168.1.100", + "port": 8766 + }, + "kitchen": { + "host": "192.168.1.101", + "port": 8766 + } + } +} +``` + +### Default + +If no configuration is provided, the server defaults to `localhost:8766` (for local development). + +## Installation + +```bash +pip install -e . +``` + +## Usage + +Run the server: + +```bash +python -m javis_slave_mcp +``` + +Or with a custom config: + +```bash +export JAVIS_SLAVE_DEVICES="my_device=192.168.1.100:8766" +python -m javis_slave_mcp +``` + +## MCP Tools + +All tools take `device_id` (default `"localhost"`) unless noted otherwise. + +**Status & discovery** + +- `slave_list_devices()` — List all configured slave devices +- `slave_get_status(device_id)` — Get current device status (incl. `muted`) +- `slave_get_info(device_id)` — Get device information +- `slave_get_ip_address(device_id)` — IP of the slave as last reported by the master +- `slave_get_connected_devices(device_id)` — All slaves known to the master, with IPs + +**Microphone** + +- `slave_mute_mic(device_id)` — Mute the microphone +- `slave_unmute_mic(device_id)` — Unmute the microphone +- `slave_toggle_mic(device_id)` — Toggle microphone state +- `slave_restart_audio(device_id)` — Restart the audio engine (stop + start) + +**Volume & output** + +- `slave_get_volume(device_id)` — Current master volume (0.0–1.0) +- `slave_set_volume(device_id, level)` — Set master volume +- `slave_increase_volume(device_id, amount=0.05)` — Increase volume +- `slave_decrease_volume(device_id, amount=0.05)` — Decrease volume +- `slave_mute_output(device_id)` — Mute output +- `slave_unmute_output(device_id)` — Unmute output + +**Brightness** + +- `slave_get_brightness(device_id)` — Current display brightness (0–100) +- `slave_set_brightness(device_id, level)` — Set brightness +- `slave_increase_brightness(device_id, amount=10)` — Increase brightness +- `slave_decrease_brightness(device_id, amount=10)` — Decrease brightness + +**Power** + +- `slave_sleep(device_id, delay=0)` — Suspend (delay in seconds, 0–3600) +- `slave_shutdown(device_id, delay=0)` — Power off +- `slave_restart_system(device_id, delay=0)` — Reboot the OS +- `slave_cancel_shutdown(device_id)` — Cancel a pending shutdown/restart + +## Master-Derived Tools + +`slave_get_ip_address` and `slave_get_connected_devices` route through the +slave's **authenticated pass-through** endpoints +(`GET /mcp/v1/master/slaves`, `GET /mcp/v1/master/slaves/{id}/ip`). The slave +calls the master with its own `X-Device-Id` / `Bearer` credentials, so this +works on LAN deployments where the MCP host and the slave host differ. + +## Integration with Javis + +Add to `mcp_servers.json`: + +```json +{ + "servers": { + "javis-slave": { + "command": "python", + "args": ["-m", "javis_slave_mcp"], + "env": { + "JAVIS_SLAVE_DEVICES": "device1=192.168.1.100:8766" + }, + "transport": "stdio", + "approval": "allow" + } + } +} +``` + +Now Javis can use tools like `javis_slave__slave_mute_mic` to control slave devices. + +## Implementation Details + +The slave app runs an HTTP server on port 8766 alongside the GUI and audio engine. This MCP server connects to slave instances via their HTTP APIs and exposes control tools. + +See: +- [Slave HTTP Server](../javis-slave/src/javis_slave/http_server.py) — Endpoints and models +- [Slave App Integration](../javis-slave/src/javis_slave/app.py) — HTTP server lifecycle diff --git a/javis-slave-mcp/pyproject.toml b/javis-slave-mcp/pyproject.toml new file mode 100644 index 0000000..a58362e --- /dev/null +++ b/javis-slave-mcp/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "javis-slave-mcp" +version = "0.1.0" +description = "MCP server for controlling Javis Slave devices — mute/unmute microphone, check status, restart audio engine." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +dependencies = [ + "mcp>=0.6.0,<2.0", + "httpx>=0.27", + "pydantic>=2.0", +] + +[project.scripts] +javis-slave-mcp = "javis_slave_mcp.server:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 110 +target-version = "py310" diff --git a/javis-slave-mcp/run.bat b/javis-slave-mcp/run.bat new file mode 100644 index 0000000..5c9451a --- /dev/null +++ b/javis-slave-mcp/run.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + pip install -e . +) else ( + call .venv\Scripts\activate.bat +) + +python -m javis_slave_mcp %* diff --git a/javis-slave-mcp/src/javis_slave_mcp/__init__.py b/javis-slave-mcp/src/javis_slave_mcp/__init__.py new file mode 100644 index 0000000..c219979 --- /dev/null +++ b/javis-slave-mcp/src/javis_slave_mcp/__init__.py @@ -0,0 +1,2 @@ +"""javis-slave-mcp: Expose local javis-slave instances as MCP tools for remote control.""" +__version__ = "0.1.0" diff --git a/javis-slave-mcp/src/javis_slave_mcp/__main__.py b/javis-slave-mcp/src/javis_slave_mcp/__main__.py new file mode 100644 index 0000000..65eb692 --- /dev/null +++ b/javis-slave-mcp/src/javis_slave_mcp/__main__.py @@ -0,0 +1,21 @@ +"""Entry point for javis-slave-mcp.""" +from __future__ import annotations + +import sys + +from .server import main + + +def __main__() -> int: + try: + main() + return 0 + except KeyboardInterrupt: + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(__main__()) diff --git a/javis-slave-mcp/src/javis_slave_mcp/server.py b/javis-slave-mcp/src/javis_slave_mcp/server.py new file mode 100644 index 0000000..fdd957e --- /dev/null +++ b/javis-slave-mcp/src/javis_slave_mcp/server.py @@ -0,0 +1,945 @@ +"""MCP server for controlling local Javis Slave devices.""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Optional + +import httpx +from mcp.server.fastmcp import FastMCP, Context +from pydantic import BaseModel, Field + +log = logging.getLogger("javis_slave_mcp") + +# ---- Models ---- + + +class SlaveStatus(BaseModel): + """Current status of a slave device.""" + device_id: str + device_name: str + room: Optional[str] = None + paired: bool + engine_running: bool + audio_state: str + muted: bool = False + mic_level: float = 0.0 + wake_word_score: float = 0.0 + last_error: Optional[str] = None + + +class DeviceInfo(BaseModel): + """Static device information.""" + device_id: str + device_name: str + sample_rate: int + input_device: Optional[int] = None + output_device: Optional[int] = None + wake_word_model: str + + +class ControlResult(BaseModel): + """Result of a control operation.""" + success: bool + message: str = "" + device_id: str = "" + + +class SlaveDeviceInfo(BaseModel): + """Information about a connected slave device.""" + device_id: str + name: str + room: Optional[str] = None + ip_address: Optional[str] = None + last_seen: Optional[str] = None + last_seen_ip: Optional[str] = None + created_at: Optional[str] = None + + +# ---- Client ---- + + +@dataclass +class SlaveClient: + """HTTP client for a Javis Slave device.""" + device_id: str + host: str + port: int = 8766 + + def base_url(self) -> str: + return f"http://{self.host}:{self.port}" + + async def get_status(self) -> SlaveStatus: + """Fetch current device status.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/status") + r.raise_for_status() + return SlaveStatus(**r.json()) + + async def get_device_info(self) -> DeviceInfo: + """Fetch device information.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/device-info") + r.raise_for_status() + return DeviceInfo(**r.json()) + + async def mute_mic(self) -> ControlResult: + """Mute the microphone.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/mic/mute") + r.raise_for_status() + result = r.json() + result["device_id"] = self.device_id + return ControlResult(**result) + + async def unmute_mic(self) -> ControlResult: + """Unmute the microphone.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/mic/unmute") + r.raise_for_status() + result = r.json() + result["device_id"] = self.device_id + return ControlResult(**result) + + async def toggle_mic(self) -> ControlResult: + """Toggle microphone on/off.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/mic/toggle") + r.raise_for_status() + result = r.json() + result["device_id"] = self.device_id + return ControlResult(**result) + + async def restart_audio(self) -> ControlResult: + """Restart the audio engine.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/restart") + r.raise_for_status() + result = r.json() + result["device_id"] = self.device_id + return ControlResult(**result) + + # ---- Volume Control Methods ---- + + async def get_volume(self) -> dict: + """Get current master volume (0.0-1.0).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/volume") + r.raise_for_status() + return r.json() + + async def set_volume(self, level: float) -> dict: + """Set master volume (0.0-1.0).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/volume/set", params={"level": level}) + r.raise_for_status() + return r.json() + + async def increase_volume(self, amount: float = 0.05) -> dict: + """Increase volume by amount.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/volume/increase", params={"amount": amount}) + r.raise_for_status() + return r.json() + + async def decrease_volume(self, amount: float = 0.05) -> dict: + """Decrease volume by amount.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/volume/decrease", params={"amount": amount}) + r.raise_for_status() + return r.json() + + async def mute_output(self) -> dict: + """Mute master audio output.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/volume/mute") + r.raise_for_status() + return r.json() + + async def unmute_output(self) -> dict: + """Unmute master audio output.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/volume/unmute") + r.raise_for_status() + return r.json() + + # ---- Brightness Control Methods ---- + + async def get_brightness(self) -> dict: + """Get current display brightness (0-100).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/brightness") + r.raise_for_status() + return r.json() + + async def set_brightness(self, level: int) -> dict: + """Set display brightness (0-100).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/brightness/set", params={"level": level}) + r.raise_for_status() + return r.json() + + async def increase_brightness(self, amount: int = 5) -> dict: + """Increase brightness by amount.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/brightness/increase", params={"amount": amount}) + r.raise_for_status() + return r.json() + + async def decrease_brightness(self, amount: int = 5) -> dict: + """Decrease brightness by amount.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/brightness/decrease", params={"amount": amount}) + r.raise_for_status() + return r.json() + + # ---- Power Control Methods ---- + + async def sleep(self) -> dict: + """Put system to sleep.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/power/sleep") + r.raise_for_status() + return r.json() + + async def shutdown(self, delay: int = 0) -> dict: + """Shutdown system (delay in seconds).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/power/shutdown", params={"delay": delay}) + r.raise_for_status() + return r.json() + + async def restart(self, delay: int = 0) -> dict: + """Restart system (delay in seconds).""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/power/restart", params={"delay": delay}) + r.raise_for_status() + return r.json() + + async def cancel_shutdown(self) -> dict: + """Cancel pending shutdown/restart.""" + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.post(f"{self.base_url()}/mcp/v1/power/cancel") + r.raise_for_status() + return r.json() + + async def get_connected_slaves(self) -> list[SlaveDeviceInfo]: + """Fetch list of all connected slaves from the master. + + Uses the slave's authenticated pass-through endpoint + (``GET /mcp/v1/master/slaves`` on the slave's own port). The slave + calls the master with its own ``X-Device-Id`` / Bearer credentials, + so this works whether or not the MCP host and the slave host match. + """ + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/master/slaves") + r.raise_for_status() + return [SlaveDeviceInfo(**item) for item in r.json()] + + async def get_slave_ip(self, target_device_id: str) -> dict: + """Fetch IP info for a specific slave from the master via the pass-through. + + The slave's own credentials authenticate the request to the master, so + this works whether or not the MCP host and the slave host match. + + Args: + target_device_id: The device_id of the slave whose IP to look up + (may be this same slave or a different one). + + Returns: + The ``MasterSlaveIp`` dict (``device_id``, ``ip_address``, + ``last_seen_ip``, ``last_seen``) as reported by the master. + """ + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url()}/mcp/v1/master/slaves/{target_device_id}/ip") + r.raise_for_status() + return r.json() + + +# ---- MCP Server ---- + + +def load_slave_config() -> dict[str, SlaveClient]: + """Load configured slave devices from environment or config file. + + For now, supports: + - Environment variables: JAVIS_SLAVE_DEVICES={device_id}={host}:{port},... + - Hardcoded localhost:8766 as default fallback + """ + import os + import json + + slaves: dict[str, SlaveClient] = {} + + # Try env var + env_slaves = os.environ.get("JAVIS_SLAVE_DEVICES", "") + if env_slaves: + try: + for pair in env_slaves.split(","): + device_id, hostport = pair.split("=") + host, port_str = hostport.rsplit(":", 1) + slaves[device_id] = SlaveClient( + device_id=device_id.strip(), + host=host.strip(), + port=int(port_str), + ) + except ValueError as e: + log.warning(f"Failed to parse JAVIS_SLAVE_DEVICES: {e}") + + # Try config file + config_file = os.environ.get( + "JAVIS_SLAVE_MCP_CONFIG", + os.path.expanduser("~/.javis/slave_mcp_config.json"), + ) + if os.path.exists(config_file): + try: + with open(config_file) as f: + config = json.load(f) + for device_id, device_cfg in config.get("slaves", {}).items(): + slaves[device_id] = SlaveClient( + device_id=device_id, + host=device_cfg.get("host", "127.0.0.1"), + port=device_cfg.get("port", 8766), + ) + except Exception as e: + log.warning(f"Failed to load slave config from {config_file}: {e}") + + # Default fallback: localhost + if not slaves: + log.info("No slaves configured; using localhost:8766 as default") + slaves["localhost"] = SlaveClient( + device_id="localhost", + host="127.0.0.1", + port=8766, + ) + + return slaves + + +# Global slave registry +_slaves: dict[str, SlaveClient] = {} + + +@dataclass +class AppContext: + slaves: dict[str, SlaveClient] + + +mcp = FastMCP( + name="javis-slave", + instructions=( + "Use these tools to query and control local Javis Slave devices. " + "Supported operations: status, device info, IP lookup, " + "mute/unmute/toggle microphone, restart audio engine, " + "volume get/set/increase/decrease, output mute/unmute, " + "brightness get/set/increase/decrease, " + "power sleep/shutdown/restart, and cancel shutdown. " + "Master-derived tools (slave_get_ip_address, " + "slave_get_connected_devices) route through the slave's own " + "credentials, so they work on LAN deployments where the MCP " + "host and the slave host differ." + ), +) + + +# ---- Tools ---- + + +@mcp.tool( + description="Get the current status of a Javis Slave device, including audio state and mic level." +) +def slave_get_status(device_id: str = "localhost") -> str: + """Get slave device status. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON status object with device info and current audio state. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + status = asyncio.run(_slaves[device_id].get_status()) + return status.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to get status for {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Get device information for a Javis Slave (static config like device name, sample rate, audio devices)." +) +def slave_get_info(device_id: str = "localhost") -> str: + """Get slave device info. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON device info object. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + info = asyncio.run(_slaves[device_id].get_device_info()) + return info.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to get info for {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Get the IP address of a configured Javis Slave device, as last reported by the master." +) +def slave_get_ip_address(device_id: str = "localhost") -> str: + """Get slave device IP address from master records. + + Routes through the slave's authenticated pass-through + (``GET /mcp/v1/master/slaves/{id}/ip``), so the slave presents its own + credentials to the master. This works even when the MCP host and the + slave host differ. + + Args: + device_id: The slave device ID to look up (default: localhost). + + Returns: + IP address string, or error message if not found. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + data = asyncio.run(_slaves[device_id].get_slave_ip(device_id)) + return data.get("ip_address") or data.get("last_seen_ip") or "Unknown" + except Exception as e: + log.error(f"Failed to get IP for {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Mute the microphone on a Javis Slave device." +) +def slave_mute_mic(device_id: str = "localhost") -> str: + """Mute slave microphone. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON control result. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + result = asyncio.run(_slaves[device_id].mute_mic()) + return result.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to mute mic on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Unmute the microphone on a Javis Slave device." +) +def slave_unmute_mic(device_id: str = "localhost") -> str: + """Unmute slave microphone. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON control result. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + result = asyncio.run(_slaves[device_id].unmute_mic()) + return result.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to unmute mic on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Toggle the microphone on/off on a Javis Slave device." +) +def slave_toggle_mic(device_id: str = "localhost") -> str: + """Toggle slave microphone. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON control result with new mic state. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + result = asyncio.run(_slaves[device_id].toggle_mic()) + return result.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to toggle mic on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Restart the audio engine on a Javis Slave device (useful for recovery from errors)." +) +def slave_restart_audio(device_id: str = "localhost") -> str: + """Restart slave audio engine. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + JSON control result. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + result = asyncio.run(_slaves[device_id].restart_audio()) + return result.model_dump_json(indent=2) + except Exception as e: + log.error(f"Failed to restart audio on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="List all configured Javis Slave devices." +) +def slave_list_devices() -> str: + """List all configured slave devices. + + Returns: + JSON array of device IDs. + """ + devices = list(_slaves.keys()) + return f"Configured slave devices: {devices}" + + +@mcp.tool( + description="Get list of all connected slaves with their IP addresses from the master." +) +def slave_get_connected_devices(device_id: str = "localhost") -> str: + """Get list of connected slaves with IPs from master. + + Routes through the configured slave's authenticated pass-through + (``GET /mcp/v1/master/slaves``). The slave presents its own credentials + to the master, so this works on LAN deployments where the MCP host and + the slave host differ. + + Args: + device_id: Configured slave to route the master query through + (any paired device works; default: localhost). + + Returns: + JSON array of slave device info objects with IP addresses. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + slaves_data = asyncio.run(_slaves[device_id].get_connected_slaves()) + import json + return json.dumps([s.model_dump() for s in slaves_data], indent=2) + except Exception as e: + log.error(f"Failed to get connected slaves: {e}") + return f"ERROR: {e}" + + +# ---- Volume Control Tools ---- + + +@mcp.tool( + description="Get the current master volume level of a slave device (0.0-1.0)." +) +def slave_get_volume(device_id: str = "localhost") -> str: + """Get slave device volume level. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Volume level as JSON response. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].get_volume()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to get volume from {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Set the master volume level on a slave device (0.0-1.0)." +) +def slave_set_volume(device_id: str = "localhost", level: float = 0.5) -> str: + """Set slave device volume level. + + Args: + device_id: The slave device ID (default: localhost). + level: Volume level 0.0-1.0 (default 0.5). + + Returns: + Volume control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].set_volume(level)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to set volume on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Increase the master volume on a slave device." +) +def slave_increase_volume(device_id: str = "localhost", amount: float = 0.05) -> str: + """Increase slave device volume. + + Args: + device_id: The slave device ID (default: localhost). + amount: Amount to increase 0.0-1.0 (default 0.05 = 5%). + + Returns: + Volume control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].increase_volume(amount)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to increase volume on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Decrease the master volume on a slave device." +) +def slave_decrease_volume(device_id: str = "localhost", amount: float = 0.05) -> str: + """Decrease slave device volume. + + Args: + device_id: The slave device ID (default: localhost). + amount: Amount to decrease 0.0-1.0 (default 0.05 = 5%). + + Returns: + Volume control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].decrease_volume(amount)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to decrease volume on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Mute the master audio output on a slave device." +) +def slave_mute_output(device_id: str = "localhost") -> str: + """Mute slave device audio output. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Volume control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].mute_output()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to mute output on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Unmute the master audio output on a slave device." +) +def slave_unmute_output(device_id: str = "localhost") -> str: + """Unmute slave device audio output. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Volume control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].unmute_output()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to unmute output on {device_id}: {e}") + return f"ERROR: {e}" + + +# ---- Brightness Control Tools ---- + + +@mcp.tool( + description="Get the current display brightness on a slave device (0-100)." +) +def slave_get_brightness(device_id: str = "localhost") -> str: + """Get slave device brightness level. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Brightness level as JSON response. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].get_brightness()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to get brightness from {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Set the display brightness on a slave device (0-100)." +) +def slave_set_brightness(device_id: str = "localhost", level: int = 50) -> str: + """Set slave device brightness level. + + Args: + device_id: The slave device ID (default: localhost). + level: Brightness level 0-100 (default 50). + + Returns: + Brightness control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].set_brightness(level)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to set brightness on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Increase the display brightness on a slave device." +) +def slave_increase_brightness(device_id: str = "localhost", amount: int = 5) -> str: + """Increase slave device brightness. + + Args: + device_id: The slave device ID (default: localhost). + amount: Amount to increase in % points (default 5). + + Returns: + Brightness control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].increase_brightness(amount)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to increase brightness on {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Decrease the display brightness on a slave device." +) +def slave_decrease_brightness(device_id: str = "localhost", amount: int = 5) -> str: + """Decrease slave device brightness. + + Args: + device_id: The slave device ID (default: localhost). + amount: Amount to decrease in % points (default 5). + + Returns: + Brightness control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].decrease_brightness(amount)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to decrease brightness on {device_id}: {e}") + return f"ERROR: {e}" + + +# ---- Power Control Tools ---- + + +@mcp.tool( + description="Put a slave device to sleep." +) +def slave_sleep(device_id: str = "localhost") -> str: + """Put slave device to sleep. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Power control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].sleep()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to sleep {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Shutdown a slave device." +) +def slave_shutdown(device_id: str = "localhost", delay: int = 0) -> str: + """Shutdown slave device. + + Args: + device_id: The slave device ID (default: localhost). + delay: Delay in seconds before shutdown (default 0 = immediate). + + Returns: + Power control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].shutdown(delay)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to shutdown {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Restart a slave device." +) +def slave_restart_system(device_id: str = "localhost", delay: int = 0) -> str: + """Restart slave device. + + Args: + device_id: The slave device ID (default: localhost). + delay: Delay in seconds before restart (default 0 = immediate). + + Returns: + Power control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].restart(delay)) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to restart {device_id}: {e}") + return f"ERROR: {e}" + + +@mcp.tool( + description="Cancel a pending shutdown or restart on a slave device." +) +def slave_cancel_shutdown(device_id: str = "localhost") -> str: + """Cancel pending shutdown/restart on slave device. + + Args: + device_id: The slave device ID (default: localhost). + + Returns: + Power control result as JSON. + """ + if device_id not in _slaves: + return f"ERROR: device '{device_id}' not found" + + try: + import asyncio + import json + result = asyncio.run(_slaves[device_id].cancel_shutdown()) + return json.dumps(result, indent=2) + except Exception as e: + log.error(f"Failed to cancel shutdown on {device_id}: {e}") + return f"ERROR: {e}" + + +# ---- Entry Point ---- + + +def main() -> None: + global _slaves + logging.basicConfig(level=logging.INFO) + _slaves = load_slave_config() + if _slaves: + log.info(f"Loaded {len(_slaves)} slave device(s): {list(_slaves.keys())}") + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/javis-slave/RASPBERRY_PI.md b/javis-slave/RASPBERRY_PI.md new file mode 100644 index 0000000..788db56 --- /dev/null +++ b/javis-slave/RASPBERRY_PI.md @@ -0,0 +1,162 @@ +# javis-slave on a Raspberry Pi + +Turn a Raspberry Pi with a USB microphone and a speaker into a Javis satellite: +it detects the wake word locally, streams the utterance to the Javis master on +your LAN, and plays the spoken reply. No GPU, no models beyond the tiny +openWakeWord ONNX file. + +## Hardware + +| Part | Notes | +|------|-------| +| Raspberry Pi 3B+ / 4 / 5 / Zero 2 W | 64-bit Raspberry Pi OS (Bookworm or newer). Zero 2 W works headless; wake-word inference uses ~15 % of one core. | +| USB microphone or USB sound card + mic | Most USB mics only offer 44.1/48 kHz — the slave resamples to 16 kHz automatically. Conference-style mics (e.g. Jabra, Anker) give the best wake-word range. | +| Speaker | 3.5 mm jack, HDMI, USB DAC or an I²S HAT. | +| Optional: official 7" touch display | Use `setup-pi.sh --gui` for the same touch UI as the Surface build; brightness control works via `/sys/class/backlight`. | + +## Install (5 minutes) + +```bash +# On the Pi +git clone javis && cd javis/javis-slave # or scp the javis-slave folder +sudo ./setup-pi.sh # headless +# sudo ./setup-pi.sh --gui # with touch display +# add --with-vad for silero-vad end-pointing (pulls PyTorch, ~200 MB; RMS gate is used otherwise) +``` + +The script installs apt dependencies, creates `/opt/javis-slave/.venv`, +pre-downloads the wake-word models, grants the service user audio/backlight/power +permissions and installs a `systemd` unit (`javis-slave.service`, enabled but not +started). + +## Configure & pair + +1. **Pick audio devices** + + ```bash + cd /opt/javis-slave + export JAVIS_SLAVE_CONFIG_DIR=/opt/javis-slave/config + .venv/bin/python -m javis_slave --list-devices + ``` + + Note the index of your mic (`max_input_channels > 0`) and speaker. + +2. **Enable the API on the master** (`config.yaml`), then restart Javis: + + ```yaml + api: + enabled: true + host: 0.0.0.0 + port: 8765 + ``` + + Open the **Devices** tab → *Generate pairing code*. The tab shows the + master's LAN address and the 6-digit code (valid 5 min, single use). + +3. **Pair and test in the foreground** + + ```bash + .venv/bin/python -m javis_slave --headless \ + --input-device 1 --output-device 0 \ + --pair 192.168.1.10:8765 123456 --name Kueche + ``` + + Say the wake word (default *hey jarvis*) — you should hear the ascending + two-tone chime, speak, and hear the reply. Device settings are persisted in + `$JAVIS_SLAVE_CONFIG_DIR/slave_config.json`. Stop with Ctrl+C. + +4. **Run as a service** + + ```bash + sudo systemctl start javis-slave + journalctl -u javis-slave -f + ``` + + Assign a room in the master's Devices tab (e.g. `Küche`); the slave picks it + up within a minute and every utterance is tagged with it for the LLM. + + With `setup-pi.sh --gui` the system service stays disabled and the touch UI + autostarts in the desktop session instead (`~/.config/autostart/javis-slave.desktop`, + `--kiosk` = fullscreen, hidden cursor; Esc leaves fullscreen, F11 toggles). + +Logs: `journalctl -u javis-slave` (service) and +`$JAVIS_SLAVE_CONFIG_DIR/logs/slave.log` (rotating, both modes). + +## Audible feedback (no screen needed) + +| Sound | Meaning | +|-------|---------| +| Two rising tones | Wake word heard — speak now | +| Two falling low tones | Something failed (master unreachable, mic error) | +| Spoken error phrase | The master received the request but its turn failed (STT/LLM); it sends its configured `agent.error_phrase` as speech | + +## Remote control from the master + +The slave runs a small HTTP control server on port 8766 (`/mcp/v1/...`) that +the [javis-slave-mcp](../javis-slave-mcp/README.md) server exposes as tools, so +you can say *"mach die Küche leiser"*. On Linux the endpoints map to: + +| Endpoint | Backend | +|----------|---------| +| `/mcp/v1/volume/*` | `pactl` (PipeWire/Pulse) → falls back to `amixer` (ALSA) | +| `/mcp/v1/brightness/*` | `/sys/class/backlight/*/brightness` (udev rule from `setup-pi.sh`) | +| `/mcp/v1/power/*` | `systemctl suspend`, `shutdown` via a scoped sudoers entry | +| `/mcp/v1/mic/*`, `/status`, `/restart` | Engine mute / status / restart | + +## CLI reference + +``` +javis-slave --headless service mode, no window +javis-slave --fullscreen | --kiosk GUI fullscreen / fullscreen + hidden cursor +javis-slave --pair HOST:PORT CODE --name N pair (works with and without --headless) +javis-slave --list-devices print sounddevice table +javis-slave --input-device N --output-device N --input-gain G --api-port P + persist settings and start +``` + +Environment: `JAVIS_SLAVE_CONFIG_DIR` overrides the config folder +(default `~/.javis-slave`). + +## Troubleshooting + +**No devices listed / "Error querying device"** — `sudo apt install libportaudio2` +and make sure the user is in the `audio` group (`groups`; log out/in after +`setup-pi.sh`). + +**Mic opens at 48 kHz, wake word never fires** — normal; the log line +`mic started @ 48000Hz→16000Hz` confirms resampling is active. Raise +`--input-gain 2.0` if the level is low; check with `arecord -d 3 test.wav`. +Stereo-only USB cards are opened with both channels and mixed to mono. + +**USB mic unplugged / stops delivering audio** — after ~2 minutes of silence +from the device the engine logs `reopening device` and re-opens it; if the +device is gone the engine start is retried every 10 s. + +**"Master … not reachable"** — the master must have `api.enabled: true` and its +Windows firewall must allow inbound TCP 8765 (Javis is a Python process; allow +`python.exe` on private networks). + +**Wake word too sensitive / not sensitive enough** — change +`wake_word.threshold` on the *master*; slaves pull it on the next config poll +(≤ 60 s). + +**Reply is silent but text appears in the master's log** — the master's TTS has +no voice profile with reference clips; add one in the Voices tab. Playback +device problems show up as `speaker does not support … Hz` (auto-resampled) or +an ALSA error in `journalctl`. + +**Service dies at boot** — USB audio may enumerate after the service starts; +`Restart=always` + `RestartSec=5` recover automatically. The engine also +retries every 10 s while the mic is unavailable. + +**Pi Zero 2 W CPU load** — skip `--with-vad`; the RMS gate is much cheaper than +Silero on a single core. Keep only one wake-word model. + +## Uninstall + +```bash +sudo systemctl disable --now javis-slave +sudo rm /etc/systemd/system/javis-slave.service /etc/sudoers.d/javis-slave \ + /etc/udev/rules.d/90-javis-backlight.rules +sudo rm -rf /opt/javis-slave +``` diff --git a/javis-slave/README.md b/javis-slave/README.md new file mode 100644 index 0000000..50590b1 --- /dev/null +++ b/javis-slave/README.md @@ -0,0 +1,89 @@ +# javis-slave + +Satellite app for [Javis](../README.md). Runs on a tablet (designed for a +Surface Pro 7) **or headless on a Raspberry Pi**, listens for the wake word +locally, and sends recorded utterances to the Javis master over the LAN for +transcription, LLM reasoning and speech synthesis. The reply is shown on +screen (GUI mode) and played back through the device's speakers. + +→ **Raspberry Pi guide:** [RASPBERRY_PI.md](RASPBERRY_PI.md) + +## How it works + +1. **Pairing** — On the master, open the **Devices** tab and click + *Generate pairing code*. On the slave, enter the master's IP, port and the + 6-digit code (setup screen, or `--pair HOST:PORT CODE`). The master issues + a permanent per-device API key and the device appears in its Devices list. +2. **Room assignment** — In the master's Devices tab, type a room name for + the device (e.g. `Küche`) and click *Save room*. The slave picks it up on + its next config refresh; every utterance from this device is tagged + `[Room: Küche]` for the LLM. +3. **Wake word** — The slave uses the same openWakeWord model name the master + is configured with (sent during pairing / config refresh) and detects the + wake word fully locally. Built-in models ship with openWakeWord; a custom + model (e.g. `rufus`, see [docs/WAKE_WORD.md](../docs/WAKE_WORD.md)) is + downloaded from the master on first use. A short rising chime confirms + detection. +4. **Turns** — After the wake word, the utterance is recorded locally + (Silero VAD or RMS end-pointing), uploaded as WAV to `POST /api/v1/turn`, + and the master's reply text + synthesized speech are returned and played. + If the master's turn fails it returns its spoken error phrase instead, so a + screenless satellite still gets audible feedback; connection problems play + a low falling tone. + +## Setup + +This folder is fully standalone — it can be copied on its own to a +satellite device without the rest of the Javis repo. It vendors its own small +copy of the audio stack (`javis_slave/audio_io/`), so there's no cross-project +dependency. + +On the **master** (`config.yaml`): + +```yaml +api: + enabled: true + host: 0.0.0.0 + port: 8765 +``` + +On the **slave**: + +```bat +run.bat :: Windows tablet (installs the GUI extra) +``` + +```bash +./run.sh # Linux with display +JAVIS_SLAVE_HEADLESS=1 ./run.sh --headless # Linux without display +sudo ./setup-pi.sh # Raspberry Pi: apt deps + venv + systemd service +``` + +The first run creates `.venv` and installs the dependencies declared in +`pyproject.toml`. Optional extras: `gui` (PySide6/qasync, needed for the +window) and `vad` (silero-vad + PyTorch; without it an RMS energy gate is used). + +### Command line + +``` +javis-slave touch GUI +javis-slave --fullscreen | --kiosk fullscreen / fullscreen + hidden cursor +javis-slave --headless no window (services, Pi) +javis-slave --pair 192.168.1.10:8765 123456 --name Kueche +javis-slave --list-devices show mic/speaker indices +javis-slave --input-device 1 --output-device 0 --input-gain 1.5 --api-port 8766 +``` + +Settings are stored in `%LOCALAPPDATA%\JavisSlave\slave_config.json` +(Windows) or `~/.javis-slave/slave_config.json` (Linux); override the folder +with `JAVIS_SLAVE_CONFIG_DIR`. A rotating log is written to `/logs/slave.log`. + +## Notes + +- Transport is plain HTTP — use on a trusted LAN only. +- Microphones/speakers that don't support 16 kHz are opened at their native + rate and resampled automatically (common for USB audio on Linux). +- System control endpoints (volume / brightness / power, see + [javis-slave-mcp](../javis-slave-mcp/README.md)) work on Windows (pycaw, + screen-brightness-control) and Linux (`pactl`/`amixer`, `/sys/class/backlight`, + `systemctl`). diff --git a/javis-slave/javis-slave.service b/javis-slave/javis-slave.service new file mode 100644 index 0000000..de5bb7f --- /dev/null +++ b/javis-slave/javis-slave.service @@ -0,0 +1,25 @@ +[Unit] +Description=Javis Slave (headless voice satellite) +Documentation=file:///opt/javis-slave/RASPBERRY_PI.md +After=network-online.target sound.target +Wants=network-online.target + +[Service] +Type=simple +# Adjust User/paths if you installed somewhere else (setup-pi.sh fills these in). +User=pi +WorkingDirectory=/opt/javis-slave +Environment=JAVIS_SLAVE_CONFIG_DIR=/opt/javis-slave/config +# PipeWire/PulseAudio (user session) is not available to a system service; +# PortAudio falls back to ALSA, which is what we want on a Pi. +Environment=PULSE_SERVER=none +ExecStart=/opt/javis-slave/.venv/bin/python -m javis_slave --headless +Restart=always +RestartSec=5 +# Give the mic/USB audio time to enumerate after boot before the first retry. +TimeoutStartSec=90 +# Slightly higher priority for glitch-free capture on small boards. +Nice=-5 + +[Install] +WantedBy=multi-user.target diff --git a/javis-slave/pyproject.toml b/javis-slave/pyproject.toml new file mode 100644 index 0000000..c97020e --- /dev/null +++ b/javis-slave/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "javis-slave" +version = "0.1.0" +description = "Touch-optimized satellite companion for Javis — local wake word, remote brain." +readme = "README.md" +requires-python = ">=3.10,<3.14" +license = { text = "MIT" } +dependencies = [ + # Master API client + "httpx>=0.27", + # HTTP server for MCP control (plain uvicorn: no uvloop/httptools builds on small ARM boards) + "fastapi>=0.104", + "uvicorn>=0.24", + "pydantic>=2.0", + # Windows system control (volume, brightness, power) — Linux uses amixer/pactl, + # /sys/class/backlight and systemctl, no extra packages. + "pycaw>=20220226; sys_platform == 'win32'", + "screen-brightness-control>=0.20; sys_platform == 'win32'", + # Audio stack (vendored copy of javis.audio_io — standalone, no repo dependency) + "numpy>=1.24,<2.0", + "sounddevice>=0.4.6", + "soundfile>=0.12", + "scipy>=1.11", + "openwakeword>=0.6.0", + "onnxruntime>=1.17", +] + +[project.optional-dependencies] +# Touch UI (Surface / Pi with display). Not needed for `--headless`. +gui = [ + "PySide6>=6.6", + "qasync>=0.27", +] +# Neural end-pointing. Pulls in PyTorch (~200 MB on aarch64); without it the +# engine falls back to an RMS energy gate automatically. +vad = [ + "silero-vad>=5.1", +] + +[project.scripts] +javis-slave = "javis_slave.__main__:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 110 +target-version = "py310" diff --git a/javis-slave/run.bat b/javis-slave/run.bat new file mode 100644 index 0000000..b740197 --- /dev/null +++ b/javis-slave/run.bat @@ -0,0 +1,16 @@ +@echo off +setlocal +cd /d "%~dp0" + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + call .venv\Scripts\activate.bat + python -m pip install --upgrade pip + rem gui = PySide6/qasync for the touch UI; vad = silero-vad (+PyTorch) for better end-pointing + pip install -e ".[gui,vad]" +) else ( + call .venv\Scripts\activate.bat +) + +python -m javis_slave %* diff --git a/javis-slave/run.sh b/javis-slave/run.sh new file mode 100755 index 0000000..cf7ffa5 --- /dev/null +++ b/javis-slave/run.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Usage: ./run.sh [javis-slave args] +# JAVIS_SLAVE_HEADLESS=1 ./run.sh --headless # skip the Qt/PySide6 install (Pi without display) +# JAVIS_SLAVE_VAD=1 ./run.sh # also install silero-vad (+ PyTorch) +set -euo pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +if [[ ! -x ".venv/bin/python" ]]; then + if ! command -v python3 >/dev/null 2>&1; then + echo "Error: python3 is required." >&2 + exit 1 + fi + + python_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + case "$python_version" in + 3.10|3.11|3.12|3.13) ;; + *) echo "Error: javis-slave requires Python 3.10–3.13 (found $python_version)." >&2; exit 1 ;; + esac + + extras="" + if [[ "${JAVIS_SLAVE_HEADLESS:-0}" != "1" ]]; then extras="gui"; fi + if [[ "${JAVIS_SLAVE_VAD:-0}" == "1" ]]; then extras="${extras:+$extras,}vad"; fi + + echo "Creating virtual environment (extras: ${extras:-none})..." + python3 -m venv .venv + .venv/bin/python -m pip install --upgrade pip + if [[ -n "$extras" ]]; then + .venv/bin/python -m pip install -e ".[${extras}]" + else + .venv/bin/python -m pip install -e . + fi +fi + +exec .venv/bin/python -m javis_slave "$@" \ No newline at end of file diff --git a/javis-slave/setup-pi.sh b/javis-slave/setup-pi.sh new file mode 100644 index 0000000..ad4ef47 --- /dev/null +++ b/javis-slave/setup-pi.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# One-shot Raspberry Pi installer for javis-slave (headless or with touch display). +# +# sudo ./setup-pi.sh # headless satellite (mic + speaker) +# sudo ./setup-pi.sh --gui # Pi with official touch display +# sudo ./setup-pi.sh --with-vad # also install silero-vad (+PyTorch, ~200 MB) +# +# Idempotent: re-running upgrades the venv and rewrites the service unit. +set -euo pipefail + +INSTALL_DIR="${INSTALL_DIR:-/opt/javis-slave}" +SERVICE_USER="${SERVICE_USER:-${SUDO_USER:-pi}}" +WITH_GUI=0 +WITH_VAD=0 +for arg in "$@"; do + case "$arg" in + --gui) WITH_GUI=1 ;; + --with-vad) WITH_VAD=1 ;; + -h|--help) sed -n '2,9p' "$0"; exit 0 ;; + *) echo "unknown option: $arg" >&2; exit 2 ;; + esac +done + +if [[ $EUID -ne 0 ]]; then + echo "Run with sudo (installs apt packages and a systemd unit)." >&2 + exit 1 +fi + +SRC_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +echo "==> apt packages" +apt-get update -qq +# portaudio: sounddevice; libsndfile: soundfile; alsa-utils: amixer/aplay/arecord; +# libatomic1 + libopenblas: onnxruntime/numpy wheels on aarch64. +apt-get install -y -qq python3 python3-venv python3-dev libportaudio2 libsndfile1 \ + alsa-utils libatomic1 libopenblas0 rsync +if [[ $WITH_GUI -eq 1 ]]; then + # Qt platform plugin deps for PySide6 wheels on Raspberry Pi OS + colour emoji + # glyphs used by the orb/buttons (🎤 🔊 ⚙). + apt-get install -y -qq libxcb-cursor0 libxkbcommon-x11-0 libegl1 libgl1 libglib2.0-0 \ + libfontconfig1 libdbus-1-3 fonts-noto-color-emoji +fi + +echo "==> copying to ${INSTALL_DIR}" +mkdir -p "$INSTALL_DIR" +if [[ "$SRC_DIR" != "$INSTALL_DIR" ]]; then + rsync -a --delete --exclude .venv --exclude config --exclude '__pycache__' \ + "$SRC_DIR"/ "$INSTALL_DIR"/ +fi +mkdir -p "$INSTALL_DIR/config" +chown -R "$SERVICE_USER":"$SERVICE_USER" "$INSTALL_DIR" + +echo "==> python venv" +extras="" +[[ $WITH_GUI -eq 1 ]] && extras="gui" +[[ $WITH_VAD -eq 1 ]] && extras="${extras:+$extras,}vad" +sudo -u "$SERVICE_USER" bash -c " + set -e + cd '$INSTALL_DIR' + [[ -x .venv/bin/python ]] || python3 -m venv .venv + .venv/bin/python -m pip install --upgrade pip wheel + if [[ -n '$extras' ]]; then .venv/bin/python -m pip install -e '.[$extras]'; else .venv/bin/python -m pip install -e .; fi + # Pre-download openWakeWord ONNX models so the service works offline later. + .venv/bin/python -c 'from openwakeword.utils import download_models; download_models()' || true +" + +echo "==> permissions (audio, backlight)" +usermod -aG audio,video "$SERVICE_USER" +cat > /etc/udev/rules.d/90-javis-backlight.rules <<'EOF' +# Let members of 'video' set the display brightness (javis-slave /mcp/v1/brightness). +SUBSYSTEM=="backlight", ACTION=="add", RUN+="/bin/chgrp video /sys/class/backlight/%k/brightness", RUN+="/bin/chmod g+w /sys/class/backlight/%k/brightness" +EOF +udevadm control --reload-rules || true +# Passwordless power control for the service user (only the exact commands used). +cat > /etc/sudoers.d/javis-slave < systemd unit" +sed -e "s|^User=.*|User=$SERVICE_USER|" \ + -e "s|/opt/javis-slave|$INSTALL_DIR|g" \ + "$INSTALL_DIR/javis-slave.service" > /etc/systemd/system/javis-slave.service +systemctl daemon-reload +if [[ $WITH_GUI -eq 1 ]]; then + # With a display the touch UI runs in the user's desktop session (kiosk), + # not as a system service — both would fight over the microphone. + systemctl disable javis-slave.service >/dev/null 2>&1 || true + AUTOSTART_DIR="$(getent passwd "$SERVICE_USER" | cut -d: -f6)/.config/autostart" + sudo -u "$SERVICE_USER" mkdir -p "$AUTOSTART_DIR" + cat > "$AUTOSTART_DIR/javis-slave.desktop" < argparse.Namespace: + p = argparse.ArgumentParser( + prog="javis-slave", + description="Javis satellite: local wake word, remote brain.", + epilog=( + "Examples:\n" + " javis-slave # touch GUI (needs the [gui] extra)\n" + " javis-slave --list-devices # find your mic/speaker indices\n" + " javis-slave --headless --pair 192.168.1.10:8765 123456 --name Kueche\n" + " javis-slave --headless # Raspberry Pi service mode" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--headless", action="store_true", + help="Run without a window (Raspberry Pi / systemd). Logs to stderr.") + p.add_argument("--fullscreen", action="store_true", + help="Start the GUI fullscreen (F11 toggles, Esc leaves).") + p.add_argument("--kiosk", action="store_true", + help="Fullscreen with hidden mouse cursor for touch panels.") + p.add_argument("--pair", nargs=2, metavar=("HOST:PORT", "CODE"), + help="Pair with the master using a code from its Devices tab, then start.") + p.add_argument("--name", default=None, + help="Device name shown on the master (default: hostname). Used with --pair.") + p.add_argument("--list-devices", action="store_true", + help="Print audio devices with their indices and exit.") + p.add_argument("--input-device", type=int, default=None, metavar="N", + help="Persist microphone device index N (see --list-devices).") + p.add_argument("--output-device", type=int, default=None, metavar="N", + help="Persist speaker device index N (see --list-devices).") + p.add_argument("--input-gain", type=float, default=None, metavar="G", + help="Persist microphone gain multiplier (1.0 = unchanged).") + p.add_argument("--api-port", type=int, default=None, metavar="PORT", + help="Persist the local MCP control server port (default 8766).") + return p.parse_args(argv) + + +def _apply_device_overrides(args: argparse.Namespace) -> None: + """Persist CLI-provided audio settings so services can be configured non-interactively.""" + if all(v is None for v in (args.input_device, args.output_device, args.input_gain, args.api_port)): + return + from .config import load_config, save_config + cfg = load_config() + if args.input_device is not None: + cfg.input_device = args.input_device + if args.output_device is not None: + cfg.output_device = args.output_device + if args.input_gain is not None: + cfg.input_gain = args.input_gain + if args.api_port is not None: + cfg.api_port = args.api_port + save_config(cfg) + + +def _parse_pair(args: argparse.Namespace): + if not args.pair: + return None + import socket + hostport, code = args.pair + host, _, port_s = hostport.rpartition(":") + if not host or not port_s.isdigit(): + print(f"--pair expects HOST:PORT, got {hostport!r}", file=sys.stderr) + sys.exit(2) + name = args.name or socket.gethostname() + return host, int(port_s), code.strip(), name + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + + if args.list_devices: + from .audio_io.stream import list_audio_devices + print(list_audio_devices()) + return 0 + + _apply_device_overrides(args) + pair = _parse_pair(args) + + if args.headless: + from .headless import run_headless + return run_headless(pair) + + try: + import PySide6 # noqa: F401 + except ImportError: + print( + "PySide6 is not installed. Either install the GUI extra " + "(pip install -e '.[gui]') or run with --headless.", + file=sys.stderr, + ) + return 1 + + if pair is not None: + # Pair first (no window needed), then launch the GUI already paired. + import asyncio + from .api_client import MasterClient, MasterError + from .config import load_config, save_config + from .headless import pair_cli + + async def _do_pair() -> int: + cfg = load_config() + save_config(cfg) + client = MasterClient(cfg) + try: + await pair_cli(cfg, client, *pair) + return 0 + except MasterError as exc: + print(f"pairing failed: {exc}", file=sys.stderr) + return 1 + finally: + await client.aclose() + + rc = asyncio.run(_do_pair()) + if rc != 0: + return rc + + from .app import run_app + return run_app(fullscreen=args.fullscreen, kiosk=args.kiosk) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/javis-slave/src/javis_slave/api_client.py b/javis-slave/src/javis_slave/api_client.py new file mode 100644 index 0000000..4b7bcc5 --- /dev/null +++ b/javis-slave/src/javis_slave/api_client.py @@ -0,0 +1,286 @@ +"""HTTP client for the Javis master API.""" +from __future__ import annotations + +import asyncio +import base64 +from dataclasses import dataclass +from pathlib import Path +from typing import Awaitable, Callable, Optional + +import httpx + +from .config import SlaveConfig, WakeWordSettings +from .logging_setup import get_logger + +log = get_logger("api") + + +class MasterError(Exception): + """Raised when the master rejects a request or is unreachable.""" + + +@dataclass +class PairResult: + api_key: str + room: Optional[str] + wake_word: WakeWordSettings + + +@dataclass +class TurnResult: + user_text: str + reply_text: str + audio_wav: Optional[bytes] # decoded WAV bytes (16-bit PCM) + sample_rate: Optional[int] + error: Optional[str] = None # master-side failure; reply_text is the spoken error phrase + + +def _http_detail(r: httpx.Response) -> str: + """Best-effort human-readable detail from a FastAPI error body.""" + try: + detail = r.json().get("detail") + if detail: + return f" — {detail}" + except Exception: # noqa: BLE001 + pass + return "" + + +def _wake_word_from(payload: dict) -> WakeWordSettings: + ww = payload.get("wake_word") or {} + return WakeWordSettings( + model=str(ww.get("model", "hey_jarvis")), + threshold=float(ww.get("threshold", 0.5)), + vad_threshold=float(ww.get("vad_threshold", 0.5)), + silence_ms=int(ww.get("silence_ms", 800)), + ) + + +class MasterClient: + """Async client for the master's ``/api/v1`` endpoints.""" + + def __init__(self, cfg: SlaveConfig): + self.cfg = cfg + self._client = httpx.AsyncClient(timeout=httpx.Timeout(10.0, read=180.0)) + # Connection health tracking (Alexa-style connection monitoring) + self._connection_ok: bool = True + self._retry_count: int = 0 + self._max_retries: int = 10 + self._backoff_base: float = 1.0 # start at 1s + self._backoff_max: float = 30.0 # cap at 30s + self._pending_reconnect: bool = False + + async def aclose(self) -> None: + """Close the underlying HTTP client (release the connection pool).""" + await self._client.aclose() + + @property + def connected(self) -> bool: + """Current connection health status.""" + return self._connection_ok + + @property + def retry_count(self) -> int: + """Number of consecutive failed attempts.""" + return self._retry_count + + @property + def backoff_seconds(self) -> float: + """Current exponential backoff duration in seconds.""" + return min(self._backoff_base * (2 ** self._retry_count), self._backoff_max) + + def mark_connected(self) -> None: + """Reset retry state on successful connection.""" + if not self._connection_ok: + log.info("connection restored") + self._connection_ok = True + self._retry_count = 0 + + def mark_disconnected(self, exc: Exception) -> None: + """Record a failed attempt with exponential backoff.""" + self._connection_ok = False + self._retry_count += 1 + log.warning(f"connection lost (attempt {self._retry_count}/{self._max_retries}); " + f"reconnect in {self.backoff_seconds:.0f}s: {exc}") + + def should_reconnect(self) -> bool: + """Whether another reconnection attempt is warranted.""" + if not self._connection_ok and self._retry_count < self._max_retries: + return True + if self._retry_count >= self._max_retries: + log.error("max reconnection attempts reached; requires manual intervention") + return False + + def _headers(self) -> dict: + return { + "X-Device-Id": self.cfg.device_id, + "Authorization": f"Bearer {self.cfg.api_key}", + } + + async def _retryable_request( + self, + request: Callable[[], Awaitable[httpx.Response]], + retries: Optional[int] = None, + ) -> httpx.Response: + """Execute an HTTP request with exponential backoff reconnection. + + ``request`` is a zero-arg callable that produces a *fresh* awaitable + (e.g. ``lambda: self._client.post(...)``). A new request is built on + every attempt so a failed attempt never re-awaits a closed coroutine. + + Args: + request: Zero-arg callable producing the request awaitable. + retries: Max number of retries (default ``self._max_retries``). + """ + if retries is None: + retries = self._max_retries + last_exc: Optional[Exception] = None + for attempt in range(retries + 1): + try: + r = await request() + self.mark_connected() + return r + except httpx.HTTPError as e: + last_exc = e + self.mark_disconnected(e) + if attempt >= retries: + raise MasterError( + f"Master nach {retries + 1} Versuchen nicht erreichbar." + ) from e + wait = min(self._backoff_base * (2 ** attempt), self._backoff_max) + log.info(f"retrying in {wait:.0f}s (attempt {attempt + 1}/{retries})") + await asyncio.sleep(wait) + raise last_exc # type: ignore[misc] + + async def pair(self, host: str, port: int, pairing_code: str, + device_name: str) -> PairResult: + try: + # Pairing codes are single-use: one retry is enough, and a wrong + # IP must fail in seconds, not after the full backoff ladder. + r = await self._retryable_request( + lambda: self._client.post( + f"http://{host}:{port}/api/v1/pair", + json={ + "pairing_code": pairing_code, + "device_id": self.cfg.device_id, + "device_name": device_name, + }, + ), + retries=1, + ) + except MasterError as e: + raise e + if r.status_code == 401: + raise MasterError("Pairing-Code ungültig oder abgelaufen.") + if r.status_code == 429: + raise MasterError("Zu viele Versuche — bitte kurz warten.") + if r.status_code != 200: + raise MasterError(f"Pairing fehlgeschlagen (HTTP {r.status_code}){_http_detail(r)}.") + data = r.json() + return PairResult( + api_key=str(data["api_key"]), + room=data.get("room"), + wake_word=_wake_word_from(data), + ) + + async def fetch_config(self) -> tuple[Optional[str], WakeWordSettings]: + """Return ``(room, wake_word)`` from the master.""" + try: + r = await self._retryable_request( + lambda: self._client.get( + f"{self.cfg.base_url}/api/v1/config", headers=self._headers(), timeout=10.0, + ) + ) + except MasterError as e: + raise e + if r.status_code == 401: + raise MasterError("Gerät nicht (mehr) autorisiert — bitte neu koppeln.") + if r.status_code != 200: + raise MasterError(f"Config-Abruf fehlgeschlagen (HTTP {r.status_code}).") + data = r.json() + return data.get("room"), _wake_word_from(data) + + async def list_slaves(self) -> list[dict]: + """Return the master's slave list (``GET /api/v1/slaves``) as raw dicts. + + Uses this device's own pairing credentials, so the caller (e.g. the + MCP pass-through endpoints) need not know the master's address or the + auth header format. Bounded to a single attempt (``retries=0``) so a + down master fails fast instead of running the full 10-retry backoff — + an HTTP/MCP query must not block for minutes. + """ + try: + r = await self._retryable_request( + lambda: self._client.get( + f"{self.cfg.base_url}/api/v1/slaves", + headers=self._headers(), timeout=10.0, + ), + retries=0, + ) + except MasterError as e: + raise e + if r.status_code != 200: + raise MasterError(f"Slave-list query failed (HTTP {r.status_code}).") + data = r.json() + return data if isinstance(data, list) else [] + + async def send_turn(self, wav_bytes: bytes) -> TurnResult: + """Upload a recorded utterance; block until the master's reply.""" + try: + # The turn POST runs a full STT+LLM+TTS on the master and is NOT + # idempotent — limit retries so a flaky network can't re-execute + # (and duplicate) the same utterance many times (see review A-2). + r = await self._retryable_request( + lambda: self._client.post( + f"{self.cfg.base_url}/api/v1/turn", + headers=self._headers(), + files={"audio": ("utterance.wav", wav_bytes, "audio/wav")}, + ), + retries=1, + ) + except MasterError as e: + raise e + if r.status_code == 401: + raise MasterError("Gerät nicht (mehr) autorisiert — bitte neu koppeln.") + if r.status_code != 200: + raise MasterError(f"Anfrage fehlgeschlagen (HTTP {r.status_code}){_http_detail(r)}.") + data = r.json() + audio_b64 = data.get("tts_audio_b64") + return TurnResult( + user_text=str(data.get("user_text", "")), + reply_text=str(data.get("reply_text", "")), + audio_wav=base64.b64decode(audio_b64) if audio_b64 else None, + sample_rate=data.get("sample_rate"), + error=data.get("error") or None, + ) + + async def check_health(self, host: str, port: int) -> bool: + """Unauthenticated reachability probe (``GET /api/v1/health``), single attempt.""" + try: + r = await self._client.get(f"http://{host}:{port}/api/v1/health", timeout=5.0) + return r.status_code == 200 + except httpx.HTTPError: + return False + + async def download_wake_model(self, dest: "Path") -> bool: + """Fetch the master's custom wake-word ``.onnx`` into *dest*. + + Returns False when the master uses a built-in model (HTTP 404) or is + unreachable; never raises. + """ + try: + r = await self._client.get( + f"{self.cfg.base_url}/api/v1/wake_word/model", + headers=self._headers(), timeout=60.0, + ) + except httpx.HTTPError as exc: + log.warning(f"wake model download failed: {exc}") + return False + if r.status_code != 200 or not r.content: + return False + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_suffix(".onnx.part") + tmp.write_bytes(r.content) + tmp.replace(dest) + log.info(f"downloaded custom wake model from master → {dest} ({len(r.content)} bytes)") + return True diff --git a/javis-slave/src/javis_slave/app.py b/javis-slave/src/javis_slave/app.py new file mode 100644 index 0000000..997ccab --- /dev/null +++ b/javis-slave/src/javis_slave/app.py @@ -0,0 +1,282 @@ +"""App wiring: qasync event loop, pairing flow, engine ↔ UI bridge.""" +from __future__ import annotations + +import asyncio +import sys +from typing import Optional + +from .logging_setup import get_logger + +from .api_client import MasterClient, MasterError +from .audio_engine import AudioEngine +from .config import SlaveConfig, load_config, save_config +from .http_server import build_http_app + +log = get_logger("slave.app") + +# Re-try starting the audio engine (mic not ready / unplugged) this often. +_ENGINE_RETRY_S = 10.0 + + +def run_app(*, fullscreen: bool = False, kiosk: bool = False) -> int: + from PySide6 import QtWidgets + import qasync + + from .main_window import SlaveWindow + + app = QtWidgets.QApplication.instance() or QtWidgets.QApplication(sys.argv) + app.setApplicationName("Javis Slave") + app.setStyle("Fusion") + + loop = qasync.QEventLoop(app) + asyncio.set_event_loop(loop) + + cfg = load_config() + save_config(cfg) # persist generated device_id on first run + client = MasterClient(cfg) + window = SlaveWindow() + engine: Optional[AudioEngine] = None + engine_task: Optional[asyncio.Task] = None + http_task: Optional[asyncio.Task] = None + http_server = None + retry_task: Optional[asyncio.Task] = None + last_state: str = "idle" + last_error: Optional[str] = None + + # ── HTTP server state getters ───────────────────────────────────────── + + def _get_engine() -> Optional[AudioEngine]: + return engine + + def _get_state() -> str: + return last_state + + def _get_error() -> Optional[str]: + return last_error + + def _get_mic_level() -> tuple[float, float]: + if engine is None: + return 0.0, 0.0 + # Owned by the engine (set in _wait_for_wake), exposed publicly (A-11). + return engine.last_rms, engine.last_score + + def _get_master_client() -> MasterClient: + """Provide the shared MasterClient to the HTTP server for pass-throughs.""" + return client + + async def _restart_engine() -> None: + """Stop and restart the audio engine (wired to ``POST /mcp/v1/restart``). + + Defined *after* ``_stop_engine`` / ``_start_engine`` below so we can + reference them via closure; the handler in http_server awaits it and + converts exceptions into a 500-style ``ControlResponse(success=False)``. + """ + await _stop_engine() + await _start_engine() + + # ── HTTP server lifecycle ───────────────────────────────────────────── + + async def _start_http_server() -> None: + nonlocal http_task, http_server + if http_task is not None: + return # idempotent: don't start twice + try: + from uvicorn import Server, Config + http_app = build_http_app( + cfg, + _get_engine, _get_state, _get_error, _get_mic_level, + on_restart=_restart_engine, + master_client_getter=_get_master_client, + ) + # Bind to the configured host/port (A-6: was 127.0.0.1:8766, so + # remote MCP controllers couldn't reach it). Distinct from master:8765. + # loop="none"/lifespan="off": the qasync loop is already running. + config = Config(app=http_app, host=cfg.api_host, port=cfg.api_port, + log_level="warning", loop="none", lifespan="off") + http_server = Server(config) + http_task = asyncio.create_task(http_server.serve()) + log.info(f"HTTP server started on {cfg.api_host}:{cfg.api_port} for MCP control") + except Exception as e: # noqa: BLE001 + log.exception(f"failed to start HTTP server: {e}") + + async def _stop_http_server() -> None: + nonlocal http_task, http_server + if http_task is not None: + if http_server is not None: + http_server.should_exit = True + try: + await asyncio.wait_for(http_task, timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 + http_task.cancel() + http_task = None + http_server = None + + # ── engine lifecycle ────────────────────────────────────────────────── + + async def _start_engine() -> None: + nonlocal engine, engine_task, last_state, last_error + last_error = None + engine = AudioEngine(cfg, client) + + # Capture mic level in engine for HTTP access + def _on_mic_level(rms: float, score: float) -> None: + window.engineMicLevel.emit(rms, score) + + def _on_state(state: str) -> None: + nonlocal last_state + last_state = state + window.engineState.emit(state) + + def _on_error(msg: str) -> None: + nonlocal last_error + last_error = msg + window.engineError.emit(msg) + + engine.on_state = _on_state + engine.on_mic_level = _on_mic_level + engine.on_turn = window.engineTurn.emit + engine.on_error = _on_error + engine.on_connection = window.engineConnection.emit + engine.on_room = window.engineRoom.emit + + try: + await engine.start() + except Exception as e: # noqa: BLE001 + log.exception(f"audio engine failed to start: {e}") + window.engineError.emit( + f"Audio-Start fehlgeschlagen: {e} — neuer Versuch in {_ENGINE_RETRY_S:.0f}s") + engine = None + _schedule_engine_retry() + return + + engine_task = asyncio.create_task(engine.run()) + window.engineRoom.emit(cfg.room) + window.show_main() + + def _schedule_engine_retry() -> None: + """Retry the engine start until it succeeds (mic plugged in late, USB re-enumeration).""" + nonlocal retry_task + if retry_task is not None and not retry_task.done(): + return + + async def _retry() -> None: + await asyncio.sleep(_ENGINE_RETRY_S) + if engine is None and cfg.paired: + await _start_engine() + + retry_task = asyncio.ensure_future(_retry()) + + async def _stop_engine() -> None: + nonlocal engine, engine_task, last_state, retry_task + last_state = "idle" + if retry_task is not None: + retry_task.cancel() + retry_task = None + # The MCP control server is independent of pairing state — it keeps + # running across re-pairs, so it is NOT stopped here (only at shutdown). + if engine_task is not None: + engine_task.cancel() + try: + await engine_task + except asyncio.CancelledError: + pass + engine_task = None + if engine is not None: + await engine.stop() + engine = None + + # ── pairing flow ────────────────────────────────────────────────────── + + def _on_pair_requested(host: str, port: int, code: str, name: str) -> None: + async def _do_pair(): + import socket + name_final = name or socket.gethostname() + if not await client.check_health(host, port): + window.setup_screen.show_error( + f"Master unter {host}:{port} nicht erreichbar — IP/Port prüfen und " + "sicherstellen, dass dort api.enabled: true gesetzt ist.") + return + try: + result = await client.pair(host, port, code, name_final) + except MasterError as e: + window.setup_screen.show_error(str(e)) + return + cfg.master_host = host + cfg.master_port = port + cfg.api_key = result.api_key + cfg.device_name = name_final + cfg.room = result.room + cfg.wake_word = result.wake_word + save_config(cfg) + log.info(f"paired with master {host}:{port}") + await _start_engine() + + asyncio.ensure_future(_do_pair()) + + window.setup_screen.pairRequested.connect(_on_pair_requested) + + # ── re-pair (settings) ──────────────────────────────────────────────── + + def _on_settings() -> None: + def _repair(): + async def _do(): + await _stop_engine() + cfg.api_key = "" + save_config(cfg) + window.setup_screen.pair_btn.setEnabled(True) + window.setup_screen.status_lbl.setText("") + window.show_setup() + asyncio.ensure_future(_do()) + + window.show_settings_dialog(cfg.device_id, cfg.device_name, _repair) + + window.main_screen.settingsRequested.connect(_on_settings) + + # ── mute toggle ─────────────────────────────────────────────────────── + + def _on_mute_toggled(muted: bool) -> None: + if engine is None: + return + # Route through the engine so capture is actually paused/resumed and the + # state callbacks fire (I-3: was a raw `engine.muted = muted` write). + asyncio.ensure_future(engine.mute() if muted else engine.unmute()) + log.info(f"mute toggled: {muted}") + + window.main_screen.muteToggled.connect(_on_mute_toggled) + + # ── startup ─────────────────────────────────────────────────────────── + + # Choose the initial page *before* showing the window so the setup screen + # doesn't flash for a frame (A-17). + if cfg.paired: + window.show_main() + else: + window.show_setup() + if kiosk: + window.set_kiosk(True) + elif fullscreen: + window.showFullScreen() + else: + window.show() + + # Start the MCP control server regardless of pairing state (A-7: it was + # only started after a successful engine start, so unpaired slaves had none). + asyncio.ensure_future(_start_http_server()) + + if cfg.paired: + asyncio.ensure_future(_start_engine()) + + async def _shutdown() -> None: + await _stop_engine() + await _stop_http_server() + await client.aclose() + + with loop: + loop.run_forever() + # The Qt app has quit; finish the async teardown so the mic/HTTP + # server are released instead of being lost as a fire-and-forget task. + try: + loop.run_until_complete(_shutdown()) + except Exception: # noqa: BLE001 + log.exception("shutdown failed") + return 0 diff --git a/javis-slave/src/javis_slave/audio_engine.py b/javis-slave/src/javis_slave/audio_engine.py new file mode 100644 index 0000000..ccbb6d3 --- /dev/null +++ b/javis-slave/src/javis_slave/audio_engine.py @@ -0,0 +1,395 @@ +"""Wake-word → record → upload → playback loop. + +Uses javis-slave's own vendored audio stack (``javis_slave.audio_io``): a +self-contained copy of MicStream, WakeWordDetector, Silero VAD and +``record_utterance`` so this app has no dependency on the parent ``javis`` +package and can be deployed standalone to a satellite device. +""" +from __future__ import annotations + +import asyncio +import io +import time +from pathlib import Path +from typing import Callable, Optional + +import numpy as np + +from .audio_io.capture import record_utterance +from .audio_io.stream import MicStream, play_pcm +from .audio_io.vad import make_vad +from .audio_io.wake_word import WakeWordDetector, pretrained_names +from .api_client import MasterClient, MasterError +from .config import SlaveConfig, WakeWordSettings, config_dir, save_config +from .logging_setup import get_logger + +log = get_logger("slave.audio") + +# Refresh room/wake-word settings from the master this often. +_CONFIG_POLL_S = 60.0 +# Re-enter the wake loop if the mic yields nothing for this long. +_WAKE_STALE_S = 60.0 +# Stop an in-progress recording if no audio arrives for this long (mic disconnect). +_RECORD_STALE_S = 30.0 +# After this many consecutive silent wake-loop exits, reopen the mic device. +_MIC_RESTART_AFTER_STALE = 2 + + +def _encode_wav(audio: np.ndarray, sample_rate: int) -> bytes: + import soundfile as sf + buf = io.BytesIO() + sf.write(buf, np.asarray(audio, dtype=np.float32), sample_rate, + format="WAV", subtype="PCM_16") + return buf.getvalue() + + +def _tones(sr: int, freqs: list[float], dur: float, amp: float) -> np.ndarray: + """Concatenate short sine tones with a click-free fade-out (matches the master's chimes).""" + out = [] + n = int(sr * dur) + fade_n = max(1, int(sr * 0.008)) + for f in freqs: + t = np.linspace(0, dur, n, endpoint=False, dtype=np.float32) + w = np.sin(2 * np.pi * f * t) * amp + w[-fade_n:] *= np.linspace(1.0, 0.0, fade_n, dtype=np.float32) + out.append(w.astype(np.float32)) + return np.concatenate(out) + + +class AudioEngine: + """Runs the satellite listen loop and reports to the UI via callbacks. + + All callbacks are plain sync callables invoked from the asyncio loop — + the Qt layer bridges them to signals. + """ + + def __init__(self, cfg: SlaveConfig, client: MasterClient): + self.cfg = cfg + self.client = client + self._stop = asyncio.Event() + self.mic: Optional[MicStream] = None + self.wake: Optional[WakeWordDetector] = None + self.vad = None + self._config_task: Optional[asyncio.Task] = None + # Set when last response ended with '?'; next recording will use 3x silence timeout + self._last_response_is_question: bool = False + + # UI callbacks (assigned by the app layer). + self.on_state: Callable[[str], None] = lambda s: None + self.on_mic_level: Callable[[float, float], None] = lambda rms, score: None + self.on_turn: Callable[[str, str], None] = lambda user, reply: None + self.on_error: Callable[[str], None] = lambda msg: None + self.on_connection: Callable[[bool, int, float], None] = lambda ok, retry, backoff: None + self.on_room: Callable[[Optional[str]], None] = lambda room: None + self.muted: bool = False # Alexa-style mute state + # Serialise mute/unmute so a check-then-act race can't leave the mic + # paused while the engine believes it's unmuted (review A-3). + self._mute_lock = asyncio.Lock() + # Set while unmuted so waiters wake immediately instead of polling (A-4). + self._unmuted = asyncio.Event() + self._unmuted.set() + # Latest mic level / wake score — read by the UI (public, no _last_*). + self.last_rms: float = 0.0 + self.last_score: float = 0.0 + + # ---------------- lifecycle ---------------- + + async def start(self) -> None: + self.mic = MicStream( + sample_rate=self.cfg.sample_rate, + input_device=self.cfg.input_device, + input_gain=self.cfg.input_gain, + ) + await self.mic.start() + self.vad = make_vad(threshold=self.cfg.wake_word.vad_threshold) + self.wake = await self._load_wake(self.cfg.wake_word.model, self.cfg.wake_word.threshold) + log.info(f"audio engine ready (wake word: '{self.cfg.wake_word.model}')") + + @staticmethod + def _wake_model_dir() -> "Path": + return config_dir() / "models" + + async def _load_wake(self, name: str, threshold: float) -> WakeWordDetector: + """Build the detector; fetch a custom model from the master when missing locally.""" + model_dir = self._wake_model_dir() + if name not in pretrained_names() and not (model_dir / f"{name}.onnx").is_file(): + log.info(f"wake word '{name}' is custom — fetching model from master") + if not await self.client.download_wake_model(model_dir / f"{name}.onnx"): + raise RuntimeError( + f"custom wake word model '{name}' not available from master — " + f"copy {name}.onnx to {model_dir} manually") + return await asyncio.to_thread(WakeWordDetector, name, threshold, [model_dir]) + + async def stop(self) -> None: + self._stop.set() + self._unmuted.set() # wake any waiters so they exit + if self.mic is not None: + await self.mic.stop() + if self._config_task is not None: + self._config_task.cancel() + # Release native models / free the GPU so the process can exit cleanly. + self.wake = None + self.vad = None + self._config_task = None + self.mic = None + # ---------------- mute ---------------- + + async def mute(self) -> None: + """Mute the mic: suspend capture and stop wake/turn processing.""" + async with self._mute_lock: + self.muted = True + self._unmuted.clear() + if self.mic is not None: + self.mic.pause() + self.on_state("muted") + + async def unmute(self) -> None: + """Unmute: re-enable capture and resume the wake loop.""" + async with self._mute_lock: + self.muted = False + if self.mic is not None: + self.mic.resume() + self._unmuted.set() + self.on_state("idle") + + async def toggle_mute(self) -> bool: + """Toggle mute; returns the new muted state.""" + if self.muted: + await self.unmute() + else: + await self.mute() + return self.muted + + async def _wait_unmuted_or_stop(self) -> None: + """Block until either the engine is unmuted or a stop is requested. + + Waits on the ``_unmuted`` event (set by :meth:`unmute`/:meth:`stop`) + instead of busy-polling, so there's no 0.1s wakeup latency. If a stop + is requested while muted, this returns immediately so the outer loop + can exit. + """ + unmut = asyncio.ensure_future(self._unmuted.wait()) + stopp = asyncio.ensure_future(self._stop.wait()) + try: + done, _ = await asyncio.wait( + {unmut, stopp}, return_when=asyncio.FIRST_COMPLETED) + finally: + unmut.cancel() + stopp.cancel() + if self._stop.is_set(): + return + + async def run(self) -> None: + """Main loop: wake → record → upload → play. Never raises.""" + self._config_task = asyncio.ensure_future(self._config_poll_loop()) + # Bounded backoff so a persistently-failing turn loop doesn't hot-spin, + # but stays responsive once things recover (review A-13). + consecutive_errors = 0 + stale_exits = 0 + try: + while not self._stop.is_set(): + if self.muted: + # Idle until the API/user unmutes (or we're told to stop). + await self._wait_unmuted_or_stop() + continue + try: + woke = await self._wait_for_wake() + if not woke: + if self._stop.is_set() or self.muted: + continue + # The mic yielded nothing for _WAKE_STALE_S: USB device + # unplugged / driver stalled. Reopen instead of looping forever. + stale_exits += 1 + if stale_exits >= _MIC_RESTART_AFTER_STALE and self.mic is not None: + log.warning("mic silent for too long — reopening device") + self.on_error("Mikrofon liefert keine Daten — Gerät wird neu geöffnet …") + await self.mic.restart() + stale_exits = 0 + continue + stale_exits = 0 + await self._handle_turn() + consecutive_errors = 0 + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 + consecutive_errors += 1 + log.exception(f"slave turn failed: {e}") + self.on_error(str(e)) + await self._play_error_tone() + self.on_state("idle") + await asyncio.sleep(min(0.5 * (2 ** (consecutive_errors - 1)), 30.0)) + except asyncio.CancelledError: + pass + finally: + if self._config_task is not None: + self._config_task.cancel() + self._config_task = None + + # ---------------- phases ---------------- + + async def _wait_for_wake(self) -> bool: + assert self.wake is not None and self.mic is not None + self.on_state("idle") + self.wake.reset() + self.mic.drain() + last_pub = 0.0 + async for chunk in self.mic.chunks(stale_timeout=_WAKE_STALE_S): + if self._stop.is_set(): + return False + if self.muted: + # Muted mid-loop: drop stale audio and exit the wake phase; + # the outer loop will re-enter _wait_for_wake, where the + # callback is a no-op while paused so the loop idles. + self.mic.drain() + self.on_state("muted") + return False + score = self.wake.feed(chunk) + now = time.monotonic() + if now - last_pub > 0.1: + last_pub = now + rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0 + self.last_rms = rms + self.last_score = float(score) + self.on_mic_level(rms, float(score)) + if self.wake.triggered(score): + log.info(f"wake! score={score:.2f}") + # Same ascending two-tone as the master so the cue is familiar. + asyncio.ensure_future(self._play_chime([880.0, 1320.0], 0.06, 0.18)) + return True + if self._stop.is_set(): + return False + return False + + async def _play_chime(self, freqs: list[float], dur: float, amp: float) -> None: + """Fire-and-forget feedback tone; never raises (a screenless Pi relies on these).""" + try: + await play_pcm(_tones(self.cfg.sample_rate, freqs, dur, amp), + self.cfg.sample_rate, output_device=self.cfg.output_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"chime failed (non-critical): {exc}") + + async def _play_error_tone(self) -> None: + """Low descending two-tone: "something went wrong".""" + await self._play_chime([440.0, 330.0], 0.12, 0.2) + + async def _handle_turn(self) -> None: + assert self.mic is not None + self.on_state("listening") + # If last response was a question, triple the listening timeout + silence_override = None + if self._last_response_is_question: + base_silence = self.cfg.wake_word.silence_ms + silence_override = base_silence * 3 + log.debug(f"last response was a question; extending silence timeout to {silence_override}ms (was {base_silence}ms)") + self._last_response_is_question = False # reset flag for next turn + audio = await record_utterance( + self.cfg.sample_rate, self.cfg.wake_word.silence_ms, self.mic, self.vad, + silence_ms_override=silence_override, + stale_timeout=_RECORD_STALE_S) + if audio is None or len(audio) < 1600: + log.info("no utterance captured") + self.on_state("idle") + return + + self.on_state("thinking") + wav = _encode_wav(audio, self.cfg.sample_rate) + try: + result = await self.client.send_turn(wav) + self.on_connection(True, 0, 0.0) # (ok=True, retry_count=0, backoff=0) + self.client.mark_connected() + except MasterError as e: + self.client.mark_disconnected(e) + if self.client.should_reconnect(): + # Emit connection health with retry countdown (Alexa-style) + self.on_connection(False, self.client.retry_count, self.client.backoff_seconds) + self.on_error(f"Verbindungsfehler — Wiederverbindung in {self.client.backoff_seconds:.0f}s …") + else: + self.on_error(str(e)) + await self._play_error_tone() + self.on_state("idle") + return + + if result.error: + # Master completed the request but the turn itself failed; it + # already put the spoken error phrase into reply_text/audio. + log.warning(f"master reported turn error: {result.error}") + self.on_error(result.error) + if not result.audio_wav: + await self._play_error_tone() + + if result.user_text or result.reply_text: + self.on_turn(result.user_text, result.reply_text) + # Check if response ended with '?' to trigger extended listening on next turn + if result.reply_text.rstrip().endswith("?"): + log.debug("assistant response ends with '?'; will extend listening timeout on next turn") + self._last_response_is_question = True + + if result.audio_wav and result.sample_rate: + self.on_state("speaking") + try: + import soundfile as sf + samples, sr = sf.read(io.BytesIO(result.audio_wav), + dtype="float32", always_2d=False) + if samples.ndim > 1: + samples = samples.mean(axis=1) + await play_pcm(samples, int(sr), + output_device=self.cfg.output_device, + cancel_event=self._stop) + except Exception as e: # noqa: BLE001 + log.warning(f"reply playback failed: {e}") + self.on_state("idle") + + # ---------------- config refresh ---------------- + + async def _config_poll_loop(self) -> None: + """Periodically pull room + wake-word settings from the master.""" + while not self._stop.is_set(): + try: + room, ww = await self.client.fetch_config() + self.on_connection(True, 0, 0.0) + self.client.mark_connected() + changed = self._apply_master_settings(room, ww) + if changed: + save_config(self.cfg) + self.on_room(room) + except MasterError as e: + self.client.mark_disconnected(e) + if self.client.should_reconnect(): + self.on_connection(False, self.client.retry_count, self.client.backoff_seconds) + else: + self.on_error("Maximale Wiederverbindungsversuche erreicht. Bitte manuell neu koppeln.") + except Exception: # noqa: BLE001 + log.exception("config refresh failed unexpectedly") + try: + await asyncio.wait_for(self._stop.wait(), timeout=_CONFIG_POLL_S) + except asyncio.TimeoutError: + pass + + def _apply_master_settings(self, room: Optional[str], ww: WakeWordSettings) -> bool: + changed = False + if room != self.cfg.room: + self.cfg.room = room + self.on_room(room) + changed = True + old = self.cfg.wake_word + if (ww.model, ww.threshold, ww.vad_threshold, ww.silence_ms) != ( + old.model, old.threshold, old.vad_threshold, old.silence_ms): + log.info(f"wake-word settings updated by master: {ww}") + rebuild_model = ww.model != old.model + self.cfg.wake_word = ww + if self.wake is not None: + if rebuild_model: + # Loading may download the model — do it off the poll loop. + asyncio.ensure_future(self._swap_wake_model(ww.model, ww.threshold)) + else: + self.wake.threshold = ww.threshold + self.vad = make_vad(threshold=ww.vad_threshold) + changed = True + return changed + + async def _swap_wake_model(self, name: str, threshold: float) -> None: + try: + self.wake = await self._load_wake(name, threshold) + except Exception as e: # noqa: BLE001 + log.error(f"failed to load wake model '{name}': {e}") + self.on_error(f"Wake-Word-Modell '{name}' konnte nicht geladen werden.") diff --git a/javis-slave/src/javis_slave/audio_io/__init__.py b/javis-slave/src/javis_slave/audio_io/__init__.py new file mode 100644 index 0000000..a077ecd --- /dev/null +++ b/javis-slave/src/javis_slave/audio_io/__init__.py @@ -0,0 +1,6 @@ +"""javis-slave's own copy of the Javis audio I/O stack. + +Vendored (not imported from the parent ``javis`` package) so this subproject +can be deployed standalone to a satellite device without the rest of the +Javis repo present. +""" diff --git a/javis-slave/src/javis_slave/audio_io/capture.py b/javis-slave/src/javis_slave/audio_io/capture.py new file mode 100644 index 0000000..140567c --- /dev/null +++ b/javis-slave/src/javis_slave/audio_io/capture.py @@ -0,0 +1,83 @@ +"""Utterance capture: VAD-segments speech from a mic stream. + +Vendored from ``javis.audio_io.capture`` — decoupled from the master's +``Config`` dataclass so this module has no dependency on the ``javis`` +package; takes plain ``sample_rate``/``silence_ms`` instead. +""" +from __future__ import annotations + +from typing import Optional + +import numpy as np + +from ..logging_setup import get_logger +from .stream import MicStream +from .vad import VAD + +log = get_logger("audio.capture") + + +async def record_utterance( + sample_rate: int, + silence_ms: int, + mic: MicStream, + vad: VAD, + max_seconds: float = 20.0, + initial_grace_ms: int = 1500, + silence_ms_override: Optional[int] = None, + stale_timeout: Optional[float] = None, +) -> Optional[np.ndarray]: + """Capture audio until trailing silence (*silence_ms*) is observed. + + Args: + sample_rate: Audio sample rate in Hz. + silence_ms: Silence timeout (ms) before stopping recording. + mic: Microphone stream. + vad: Voice Activity Detector. + max_seconds: Maximum recording duration in seconds. + initial_grace_ms: Grace period before speech is expected (ms). + silence_ms_override: Override the silence timeout (ms). + When set, this value is used instead of the silence_ms parameter. + stale_timeout: If set, recording stops early after this many seconds + of silence (mic disconnect guard). + + Returns the recorded float32 mono numpy array at the mic's sample rate, or + None if no speech was detected within the grace period. + """ + # Use override if provided, otherwise use the passed silence_ms + effective_silence_ms = silence_ms_override if silence_ms_override is not None else silence_ms + chunk_ms = mic.chunk_ms + + silence_chunks_needed = max(1, effective_silence_ms // chunk_ms) + grace_chunks = max(1, initial_grace_ms // chunk_ms) + max_chunks = int(max_seconds * 1000 / chunk_ms) + + vad.reset() + collected: list[np.ndarray] = [] + silent_run = 0 + seen_speech = False + chunks_seen = 0 + + async for chunk in mic.chunks(stale_timeout=stale_timeout): + chunks_seen += 1 + collected.append(chunk) + result = vad.process(chunk) + + if result.is_speech: + seen_speech = True + silent_run = 0 + else: + silent_run += 1 + + if seen_speech and silent_run >= silence_chunks_needed: + break + if not seen_speech and chunks_seen >= grace_chunks: + log.debug("no speech during grace window") + return None + if chunks_seen >= max_chunks: + log.info("max utterance length reached") + break + + audio = np.concatenate(collected) if collected else np.zeros(0, dtype=np.float32) + log.info(f"utterance: {len(audio)/sample_rate:.2f}s") + return audio diff --git a/javis-slave/src/javis_slave/audio_io/stream.py b/javis-slave/src/javis_slave/audio_io/stream.py new file mode 100644 index 0000000..c58585b --- /dev/null +++ b/javis-slave/src/javis_slave/audio_io/stream.py @@ -0,0 +1,404 @@ +"""Async microphone capture and speaker playback. + +Vendored from ``javis.audio_io.stream`` — decoupled from the master's +``Config`` dataclass so this module has no dependency on the ``javis`` +package. +""" +from __future__ import annotations + +import asyncio +import time +from typing import AsyncIterator, Optional + +import numpy as np + +from ..logging_setup import get_logger + +log = get_logger("audio.stream") + +# Warn at most once per 10 seconds when chunks are being dropped. +_DROP_WARN_INTERVAL_S = 10.0 + + +class _StreamResampler: + """Block-wise resampler that carries filter state and phase across calls. + + ``scipy.signal.resample_poly`` is stateless and smears its filter edges + over every 30 ms block (for 44.1→16 kHz the filter is longer than the + block itself), so it is unusable on a live mic. This does a stateful FIR + low-pass (when decimating) followed by linear interpolation with the + fractional read position preserved between blocks — click-free and cheap + enough for a Pi Zero. + """ + + def __init__(self, src_rate: int, dst_rate: int): + from scipy.signal import firwin + self.src, self.dst = src_rate, dst_rate + self._step = src_rate / dst_rate + self._phase = 0.0 + self._last = 0.0 + self._taps = None + self._zi = None + if src_rate > dst_rate: + # Cut at 90 % of the target Nyquist, normalised to the source Nyquist. + cutoff = min(0.99, 0.9 * (dst_rate / 2) / (src_rate / 2)) + self._taps = firwin(63, cutoff).astype(np.float32) + self._zi = np.zeros(len(self._taps) - 1, dtype=np.float32) + + def process(self, x: np.ndarray) -> np.ndarray: + if self._taps is not None: + from scipy.signal import lfilter + x, self._zi = lfilter(self._taps, [1.0], x, zi=self._zi) + n = len(x) + if n == 0: + return x.astype(np.float32) + # Only positions up to the last real sample (n-1) can be interpolated + # now; anything in (n-1, n) needs the next block's first sample and is + # carried over as a negative phase (between ``_last`` and x[0]). + last_idx = n - 1 + if self._phase <= last_idx: + count = int((last_idx - self._phase) // self._step) + 1 + pos = self._phase + self._step * np.arange(count) + pos = pos[pos <= last_idx] + else: + pos = np.empty(0) + xp = np.arange(-1, n) + fp = np.concatenate(([self._last], x)) + out = np.interp(pos, xp, fp).astype(np.float32) + self._phase = (pos[-1] + self._step - n) if len(pos) else (self._phase - n) + self._last = float(x[-1]) + return out + + +class MicStream: + """Async iterator over fixed-size mic chunks (mono float32). + + Always yields 16 kHz (``sample_rate``) audio. If the device refuses that + rate (common for USB mics on Raspberry Pi / ALSA, which only expose 44.1 or + 48 kHz), the stream is opened at the device's native rate and resampled. + """ + + _SENTINEL = object() + + def __init__(self, sample_rate: int = 16000, input_device: Optional[int] = None, + input_gain: float = 1.0, chunk_ms: int = 30): + self.sr = sample_rate + self.device = input_device + self.gain = input_gain + self.chunk_ms = chunk_ms + self.chunk_samples = int(self.sr * chunk_ms / 1000) + self._queue: asyncio.Queue = asyncio.Queue(maxsize=128) + self._stream = None + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._paused: bool = False + # Native device rate when it differs from ``sr`` (resampling active). + self._device_sr: int = sample_rate + self._resampler: Optional[_StreamResampler] = None + # Drop-warning rate limiter. + self._drop_count: int = 0 + self._last_drop_warn: float = 0.0 + + @property + def paused(self) -> bool: + """Whether capture is temporarily suspended (mic muted).""" + return self._paused + + def _cb(self, indata, frames, time_info, status): # noqa: D401 + if status: + log.debug(f"mic status: {status}") + if self._paused: + return + # Stereo-only devices: average to mono rather than taking one channel, + # so a mic wired to the right input still works. + if indata.shape[1] > 1: + chunk = indata.mean(axis=1, dtype=np.float32) + else: + chunk = indata[:, 0].copy().astype(np.float32) + if self._resampler is not None: + chunk = self._resampler.process(chunk) + if self.gain != 1.0: + chunk *= self.gain + if self._loop and not self._loop.is_closed(): + self._loop.call_soon_threadsafe(self._enqueue, chunk) + + def _enqueue(self, chunk: np.ndarray) -> None: + # Drop oldest to keep mic latency bounded when the loop is busy. + if self._queue.full(): + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + pass + self._drop_count += 1 + now = time.monotonic() + if now - self._last_drop_warn >= _DROP_WARN_INTERVAL_S: + log.warning(f"MicStream: dropped {self._drop_count} chunk(s) (queue full)") + self._drop_count = 0 + self._last_drop_warn = now + try: + self._queue.put_nowait(chunk) + except asyncio.QueueFull: + pass + + async def start(self) -> None: + import sounddevice as sd + self._loop = asyncio.get_running_loop() + self._paused = False + native = _device_default_rate(self.device, input=True) + max_ch = _device_max_channels(self.device, input=True) + # Preferred first; then the device's native rate / channel count. ALSA + # "hw:" devices on a Pi frequently accept only one exact combination. + attempts: list[tuple[int, int]] = [(self.sr, 1)] + if native and native != self.sr: + attempts.append((native, 1)) + if max_ch and max_ch > 1: + attempts.append((self.sr, max_ch)) + if native and native != self.sr: + attempts.append((native, max_ch)) + last_exc: Optional[Exception] = None + for rate, channels in attempts: + try: + stream = sd.InputStream( + samplerate=rate, + channels=channels, + dtype="float32", + blocksize=int(rate * self.chunk_ms / 1000), + device=self.device, + callback=self._cb, + ) + stream.start() + except Exception as exc: # noqa: BLE001 + last_exc = exc + log.debug(f"mic open failed @ {rate} Hz / {channels} ch: {exc}") + continue + self._stream = stream + self._device_sr = rate + self._resampler = _StreamResampler(rate, self.sr) if rate != self.sr else None + if (rate, channels) != (self.sr, 1): + log.warning(f"mic does not support {self.sr} Hz mono; capturing at " + f"{rate} Hz / {channels} ch and converting") + log.info(f"mic started @ {rate}Hz→{self.sr}Hz, chunk={self.chunk_ms}ms, device={self.device}") + return + raise RuntimeError(f"could not open microphone (device={self.device}): {last_exc}") + + def drain(self) -> None: + """Discard all buffered chunks. Call before starting a fresh listen session + to avoid processing stale audio accumulated during upload/playback.""" + drained = 0 + while True: + try: + self._queue.get_nowait() + drained += 1 + except asyncio.QueueEmpty: + break + if drained: + log.debug(f"MicStream: drained {drained} stale chunk(s)") + + def pause(self) -> None: + """Suspend capture (mute). The PortAudio stream keeps running, but the + callback drops incoming samples so no audio is enqueued or processed. + Buffered chunks are discarded so a stale wake-word can't fire on resume.""" + self._paused = True + self.drain() + log.info("MicStream: capture paused (muted)") + + def resume(self) -> None: + """Re-enable capture after :meth:`pause`. Any audio buffered while + paused is dropped first so playback isn't re-processed.""" + self.drain() + self._paused = False + log.info("MicStream: capture resumed") + + def _drain_and_sentinel(self) -> None: + """Drain queue then enqueue the sentinel so chunks() always unblocks. + + The sentinel is *guaranteed* to be enqueued: if the queue is full the + oldest chunk is discarded and we retry, so a chunks() consumer can never + block forever waiting for it. + """ + while True: + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + break + while True: + try: + self._queue.put_nowait(self._SENTINEL) + return + except asyncio.QueueFull: + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + break + + async def stop(self, *, sentinel: bool = True) -> None: + if self._stream: + try: + self._stream.stop() + self._stream.close() + except Exception as exc: # noqa: BLE001 + # A hot-unplugged USB device raises here; nothing left to close. + log.debug(f"mic close raised (ignored): {exc}") + self._stream = None + log.info("mic stopped") + # Drain first so the sentinel is never dropped when the queue is full. + if sentinel and self._loop and not self._loop.is_closed(): + self._loop.call_soon_threadsafe(self._drain_and_sentinel) + + async def restart(self) -> None: + """Reopen the device (USB re-plug / driver stall) without ending active consumers.""" + await self.stop(sentinel=False) + self.drain() + await self.start() + + async def chunks( + self, stale_timeout: Optional[float] = None + ) -> AsyncIterator[np.ndarray]: + """Yield mic chunks. If *stale_timeout* is given and the queue stays + empty for that many seconds (possible mic disconnect), the generator + exits after logging a warning instead of blocking forever.""" + while True: + try: + if stale_timeout is not None: + chunk = await asyncio.wait_for( + self._queue.get(), timeout=stale_timeout + ) + else: + chunk = await self._queue.get() + except asyncio.TimeoutError: + log.warning( + f"MicStream: no audio for {stale_timeout:.0f}s — " + "possible microphone disconnect; re-entering wake loop" + ) + return + if chunk is self._SENTINEL: + return + yield chunk + + +def _output_channels(output_device) -> int: + """Return the channel count to use when opening an OutputStream. + + Professional audio interfaces refuse ``channels=1`` on Windows MME with + PortAudio error -9998. We query the device's actual max_output_channels + and cap at 2 (stereo); mono audio is tiled to fill. + """ + import sounddevice as sd + try: + dev_idx = output_device if output_device is not None else sd.default.device[1] + info = sd.query_devices(dev_idx) + return min(max(1, int(info["max_output_channels"])), 2) + except Exception: + return 1 + + +def _device_default_rate(device, *, input: bool) -> Optional[int]: + """Native sample rate reported by PortAudio for *device* (None if unknown).""" + import sounddevice as sd + try: + idx = device if device is not None else sd.default.device[0 if input else 1] + return int(round(float(sd.query_devices(idx)["default_samplerate"]))) + except Exception: # noqa: BLE001 + return None + + +def _device_max_channels(device, *, input: bool) -> Optional[int]: + import sounddevice as sd + try: + idx = device if device is not None else sd.default.device[0 if input else 1] + key = "max_input_channels" if input else "max_output_channels" + return int(sd.query_devices(idx)[key]) + except Exception: # noqa: BLE001 + return None + + +def _resample_ratio(src: int, dst: int) -> tuple[int, int]: + """Reduced (up, down) integers for ``scipy.signal.resample_poly``.""" + from math import gcd + g = gcd(src, dst) + return dst // g, src // g + + +def list_audio_devices() -> str: + """Human-readable device table (``--list-devices``).""" + import sounddevice as sd + return str(sd.query_devices()) + + +async def play_pcm(samples: np.ndarray, sample_rate: int, + output_device: Optional[int] = None, + cancel_event: Optional[asyncio.Event] = None) -> None: + """Play a float32 mono numpy array. Awaitable + cancellable via event. + + Writes happen on a worker thread so the asyncio loop stays responsive. + Cancellation aborts the device stream immediately rather than draining + the buffer. + """ + import sounddevice as sd + + samples = np.ascontiguousarray(np.asarray(samples, dtype=np.float32)) + if samples.ndim == 1: + samples = samples[:, np.newaxis] + + n_ch = _output_channels(output_device) + if samples.shape[1] < n_ch: + samples = np.repeat(samples, n_ch, axis=1) + + # ~20 ms blocks: short enough to react to interruption, long enough to + # avoid underruns on Windows WASAPI. + block = max(256, sample_rate // 50) + try: + stream = sd.OutputStream( + samplerate=sample_rate, + channels=n_ch, + dtype="float32", + device=output_device, + blocksize=block, + ) + except Exception as exc: # noqa: BLE001 + # Pi HDMI / cheap USB DACs often only accept 44.1/48 kHz. + native = _device_default_rate(output_device, input=False) + if native is None or native == sample_rate: + raise + log.warning(f"speaker does not support {sample_rate} Hz ({exc}); resampling to {native} Hz") + from scipy.signal import resample_poly + up, down = _resample_ratio(sample_rate, native) + samples = np.ascontiguousarray(resample_poly(samples, up, down, axis=0).astype(np.float32)) + sample_rate = native + block = max(256, sample_rate // 50) + stream = sd.OutputStream( + samplerate=sample_rate, + channels=n_ch, + dtype="float32", + device=output_device, + blocksize=block, + ) + + def _writer() -> None: + i = 0 + total = len(samples) + while i < total: + if cancel_event is not None and cancel_event.is_set(): + return + end = min(i + block, total) + stream.write(samples[i:end]) + i = end + + stream.start() + cancelled = False + try: + await asyncio.to_thread(_writer) + cancelled = bool(cancel_event is not None and cancel_event.is_set()) + finally: + try: + if cancelled: + stream.abort() + else: + stream.stop() + except Exception: # noqa: BLE001 + pass + try: + stream.close() + except Exception: # noqa: BLE001 + pass + if cancelled: + log.info("playback cancelled") diff --git a/javis-slave/src/javis_slave/audio_io/vad.py b/javis-slave/src/javis_slave/audio_io/vad.py new file mode 100644 index 0000000..1f6cdaf --- /dev/null +++ b/javis-slave/src/javis_slave/audio_io/vad.py @@ -0,0 +1,96 @@ +"""Voice activity detection using silero-vad (with graceful fallback). + +Vendored from ``javis.audio_io.vad`` — decoupled from the master's hardware +diagnostics helper so this module has no dependency on the ``javis`` package. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +import numpy as np + +from ..logging_setup import get_logger + +log = get_logger("audio.vad") + + +@dataclass +class VADResult: + is_speech: bool + probability: float + + +class VAD(Protocol): + sample_rate: int + threshold: float + + def reset(self) -> None: ... + def process(self, chunk: np.ndarray) -> VADResult: ... + + +class NullVAD: + """Fallback VAD used when torch/silero is unavailable. + + Uses a simple RMS energy gate so the satellite still works (with reduced + accuracy) instead of refusing to start. + """ + + def __init__(self, sample_rate: int = 16000, threshold: float = 0.5): + self.sample_rate = sample_rate + # Re-interpret the silero threshold (0..1) as an RMS gate. + self.threshold = max(0.005, min(0.05, threshold * 0.02)) + log.warning("Using NullVAD (RMS energy gate) — silero-vad/torch unavailable") + + def reset(self) -> None: + return + + def process(self, chunk: np.ndarray) -> VADResult: + if chunk.size == 0: + return VADResult(False, 0.0) + rms = float(np.sqrt(np.mean(np.square(chunk.astype(np.float32))))) + return VADResult(is_speech=rms >= self.threshold, probability=rms) + + +def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD: + """Construct a VAD, falling back to NullVAD if torch/silero fails to load.""" + try: + return SileroVAD(sample_rate=sample_rate, threshold=threshold) + except Exception as e: # noqa: BLE001 + log.error(f"SileroVAD unavailable: {e}") + return NullVAD(sample_rate=sample_rate, threshold=threshold) + + +class SileroVAD: + """Wraps silero-vad. Expects 16 kHz mono float32 chunks of 30 or 32 ms.""" + + def __init__(self, sample_rate: int = 16000, threshold: float = 0.5): + import torch + from silero_vad import load_silero_vad + self.sample_rate = sample_rate + self.threshold = threshold + self.model = load_silero_vad() + self._window_samples = 512 if sample_rate == 16000 else 256 + # Pre-allocated tensor to avoid per-call allocations on the hot path. + self._buf = torch.zeros(self._window_samples, dtype=torch.float32) + log.info(f"silero-vad loaded (sr={sample_rate})") + + def reset(self) -> None: + try: + self.model.reset_states() + except Exception: + pass + + def process(self, chunk: np.ndarray) -> VADResult: + """Process a single window; returns speech probability.""" + import torch + w = self._window_samples + n = len(chunk) + if n >= w: + self._buf.copy_(torch.from_numpy(np.ascontiguousarray(chunk[:w], dtype=np.float32))) + else: + self._buf[:n].copy_(torch.from_numpy(np.ascontiguousarray(chunk, dtype=np.float32))) + self._buf[n:].zero_() + with torch.inference_mode(): + prob = float(self.model(self._buf, self.sample_rate).item()) + return VADResult(is_speech=prob >= self.threshold, probability=prob) diff --git a/javis-slave/src/javis_slave/audio_io/wake_word.py b/javis-slave/src/javis_slave/audio_io/wake_word.py new file mode 100644 index 0000000..6269205 --- /dev/null +++ b/javis-slave/src/javis_slave/audio_io/wake_word.py @@ -0,0 +1,109 @@ +"""Wake word detection using openWakeWord. + +Vendored from ``javis.audio_io.wake_word`` (only the logger import differs) +so javis-slave has no dependency on the parent ``javis`` package. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Iterable, Optional + +import numpy as np + +from ..logging_setup import get_logger + +log = get_logger("audio.wake") + + +def pretrained_names() -> set[str]: + try: + import openwakeword + return set(openwakeword.MODELS.keys()) + except Exception: # noqa: BLE001 + return {"alexa", "hey_mycroft", "hey_jarvis", "hey_rhasspy", "timer", "weather"} + + +def resolve_wake_model(name: str, search_dirs: Iterable[Path] = ()) -> tuple[str, str, Optional[Path]]: + """Map a model name to ``(model_arg, prediction_key, custom_path)``. + + Built-in names pass through; a custom ``.onnx`` (path or stem found in + *search_dirs*) is keyed by its filename stem, as openWakeWord does. + """ + name = (name or "hey_jarvis").strip() + p = Path(name).expanduser() + if p.suffix.lower() == ".onnx" and p.is_file(): + return str(p), p.stem, p + stem = p.stem if p.suffix else name + for d in search_dirs: + cand = Path(d) / f"{stem}.onnx" + if cand.is_file(): + return str(cand), stem, cand + if name not in pretrained_names(): + raise FileNotFoundError( + f"wake word model '{name}' is not built-in and no {stem}.onnx was found in " + f"{[str(d) for d in search_dirs]}") + return name, name, None + + +class WakeWordDetector: + """openWakeWord wrapper. Feeds 16 kHz int16 frames; emits scores per chunk.""" + + OWW_FRAME_SAMPLES = 1280 # openWakeWord expects 80 ms @ 16 kHz + + def __init__(self, model_name: str = "hey_jarvis", threshold: float = 0.5, + search_dirs: Iterable[Path] = ()): + from openwakeword.model import Model + from openwakeword.utils import download_models + # Ensure ONNX models are present (tflite_runtime has no Windows wheels). + try: + download_models() + except Exception as e: # noqa: BLE001 + log.warning(f"openWakeWord model download skipped: {e}") + model_arg, key, custom = resolve_wake_model(model_name, search_dirs) + self.model = Model( + wakeword_models=[model_arg], + inference_framework="onnx", + ) + self.model_name = key + self.custom_path: Optional[Path] = custom + self.threshold = threshold + # Single pre-allocated 80 ms int16 frame + fill index — avoids per-chunk + # list churn and np.concatenate calls in the wake-word hot path. + self._frame = np.empty(self.OWW_FRAME_SAMPLES, dtype=np.int16) + self._fill = 0 + log.info(f"openWakeWord loaded ({model_arg}, key={key}, threshold={threshold})") + + def reset(self) -> None: + self._fill = 0 + try: + self.model.reset() + except Exception: + pass + + def feed(self, chunk_f32: np.ndarray) -> float: + """Feed an arbitrary-length mono float32 chunk; return max score across complete frames.""" + # Convert to int16 in one allocation (oww expectation). + scaled = chunk_f32 * 32768.0 + np.clip(scaled, -32768.0, 32767.0, out=scaled) + i16 = scaled.astype(np.int16, copy=False) + + max_score = 0.0 + pos = 0 + n = len(i16) + frame_size = self.OWW_FRAME_SAMPLES + while pos < n: + free = frame_size - self._fill + take = min(free, n - pos) + self._frame[self._fill:self._fill + take] = i16[pos:pos + take] + self._fill += take + pos += take + if self._fill == frame_size: + preds = self.model.predict(self._frame) + score = float(preds.get(self.model_name, 0.0)) + if score > max_score: + max_score = score + self._fill = 0 + return max_score + + def triggered(self, score: float) -> bool: + return score >= self.threshold diff --git a/javis-slave/src/javis_slave/config.py b/javis-slave/src/javis_slave/config.py new file mode 100644 index 0000000..74f0202 --- /dev/null +++ b/javis-slave/src/javis_slave/config.py @@ -0,0 +1,90 @@ +"""Slave-local configuration: master address, credentials, audio devices. + +Persisted to ``%LOCALAPPDATA%/JavisSlave/slave_config.json`` (or +``~/.javis-slave/slave_config.json`` on non-Windows). Set +``JAVIS_SLAVE_CONFIG_DIR`` to override (useful for systemd services). +""" +from __future__ import annotations + +import json +import os +import uuid +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Optional + + +def config_dir() -> Path: + override = os.environ.get("JAVIS_SLAVE_CONFIG_DIR") + if override: + return Path(override).expanduser() + base = os.environ.get("LOCALAPPDATA") + if base: + return Path(base) / "JavisSlave" + return Path.home() / ".javis-slave" + + +def config_path() -> Path: + return config_dir() / "slave_config.json" + + +@dataclass +class WakeWordSettings: + """Wake-word block received from the master.""" + model: str = "hey_jarvis" + threshold: float = 0.5 + vad_threshold: float = 0.5 + silence_ms: int = 800 + + +@dataclass +class SlaveConfig: + device_id: str = field(default_factory=lambda: str(uuid.uuid4())) + device_name: str = "" + master_host: str = "" + master_port: int = 8765 + api_key: str = "" + room: Optional[str] = None # cached; refreshed from master + input_device: Optional[int] = None + output_device: Optional[int] = None + input_gain: float = 1.0 + sample_rate: int = 16000 + wake_word: WakeWordSettings = field(default_factory=WakeWordSettings) + # Local MCP control server bind (review A-6: was hardcoded to 127.0.0.1, + # unreachable by remote controllers). 0.0.0.0 lets other devices reach it. + api_host: str = "0.0.0.0" + api_port: int = 8766 + + @property + def paired(self) -> bool: + return bool(self.master_host and self.api_key) + + @property + def base_url(self) -> str: + return f"http://{self.master_host}:{self.master_port}" + + +def load_config() -> SlaveConfig: + path = config_path() + cfg = SlaveConfig() + if path.exists(): + try: + raw = json.loads(path.read_text(encoding="utf-8")) + ww_raw = raw.pop("wake_word", {}) or {} + for key, val in raw.items(): + if hasattr(cfg, key) and key != "wake_word": + setattr(cfg, key, val) + for key, val in ww_raw.items(): + if hasattr(cfg.wake_word, key): + setattr(cfg.wake_word, key, val) + except Exception: # noqa: BLE001 + pass + return cfg + + +def save_config(cfg: SlaveConfig) -> None: + path = config_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + tmp.write_text(json.dumps(asdict(cfg), indent=2), encoding="utf-8") + tmp.replace(path) diff --git a/javis-slave/src/javis_slave/headless.py b/javis-slave/src/javis_slave/headless.py new file mode 100644 index 0000000..1a2c5bf --- /dev/null +++ b/javis-slave/src/javis_slave/headless.py @@ -0,0 +1,183 @@ +"""Headless runtime: wake word → master → speaker, without Qt. + +Used on displayless satellites (Raspberry Pi + mic + speaker) via +``python -m javis_slave --headless``. Reuses :class:`AudioEngine`, +:class:`MasterClient` and the MCP control HTTP server exactly like the GUI +app; state and errors go to the log instead of the screen. +""" +from __future__ import annotations + +import asyncio +import signal +import sys +from typing import Optional + +from .api_client import MasterClient, MasterError +from .audio_engine import AudioEngine +from .config import SlaveConfig, load_config, save_config +from .http_server import build_http_app +from .logging_setup import get_logger + +log = get_logger("slave.headless") + +# Re-try the initial engine start (mic not ready at boot, master down) this often. +_START_RETRY_S = 10.0 + + +async def pair_cli(cfg: SlaveConfig, client: MasterClient, host: str, port: int, + code: str, name: str) -> None: + """Pair from the command line and persist credentials.""" + if not await client.check_health(host, port): + raise MasterError( + f"Master at {host}:{port} not reachable — is api.enabled: true in its config.yaml?") + result = await client.pair(host, port, code, name) + cfg.master_host = host + cfg.master_port = port + cfg.api_key = result.api_key + cfg.device_name = name + cfg.room = result.room + cfg.wake_word = result.wake_word + save_config(cfg) + log.info(f"paired with master {host}:{port} as {name!r} (room={result.room!r})") + + +async def run_headless_async(cfg: SlaveConfig, client: MasterClient) -> int: + if not cfg.paired: + log.error("device is not paired. Run: python -m javis_slave --pair HOST:PORT CODE --name NAME") + return 2 + + engine: Optional[AudioEngine] = None + engine_task: Optional[asyncio.Task] = None + state = {"value": "idle", "error": None} + stop = asyncio.Event() + + def _on_state(s: str) -> None: + if s != state["value"]: + state["value"] = s + log.info(f"state -> {s}") + + def _on_error(msg: str) -> None: + state["error"] = msg + log.error(msg) + + def _on_turn(user: str, reply: str) -> None: + log.info(f"user: {user}") + log.info(f"javis: {reply}") + + def _on_room(room) -> None: + log.info(f"room: {room or '(none)'}") + + def _on_connection(ok: bool, retry: int, backoff: float) -> None: + if not ok: + log.warning(f"master unreachable (attempt {retry}, next try in {backoff:.0f}s)") + + async def _start_engine() -> None: + nonlocal engine, engine_task + while not stop.is_set(): + eng = AudioEngine(cfg, client) + eng.on_state = _on_state + eng.on_error = _on_error + eng.on_turn = _on_turn + eng.on_room = _on_room + eng.on_connection = _on_connection + try: + await eng.start() + except Exception as exc: # noqa: BLE001 + log.error(f"audio engine failed to start ({exc}); retrying in {_START_RETRY_S:.0f}s") + try: + await asyncio.wait_for(stop.wait(), timeout=_START_RETRY_S) + except asyncio.TimeoutError: + continue + return + engine = eng + engine_task = asyncio.create_task(eng.run()) + log.info(f"listening for wake word '{cfg.wake_word.model}' (room={cfg.room!r})") + return + + async def _stop_engine() -> None: + nonlocal engine, engine_task + if engine_task is not None: + engine_task.cancel() + try: + await engine_task + except asyncio.CancelledError: + pass + engine_task = None + if engine is not None: + await engine.stop() + engine = None + + async def _restart() -> None: + await _stop_engine() + await _start_engine() + + # MCP control server (same endpoints as the GUI app). + http_task: Optional[asyncio.Task] = None + server = None + try: + from uvicorn import Config as UvConfig, Server + http_app = build_http_app( + cfg, + lambda: engine, lambda: state["value"], lambda: state["error"], + lambda: (engine.last_rms, engine.last_score) if engine else (0.0, 0.0), + on_restart=_restart, + master_client_getter=lambda: client, + ) + server = Server(UvConfig(app=http_app, host=cfg.api_host, port=cfg.api_port, + log_level="warning", loop="none", lifespan="off")) + http_task = asyncio.create_task(server.serve()) + log.info(f"MCP control server on http://{cfg.api_host}:{cfg.api_port}") + except Exception as exc: # noqa: BLE001 + log.error(f"HTTP server not started: {exc}") + + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, stop.set) + except (NotImplementedError, RuntimeError): + pass # Windows: fall back to KeyboardInterrupt + + await _start_engine() + try: + await stop.wait() + except (KeyboardInterrupt, asyncio.CancelledError): + # Windows has no add_signal_handler: asyncio.run cancels us on Ctrl+C. + # Swallow it so the teardown below still runs. + pass + log.info("shutting down …") + await _stop_engine() + if server is not None: + server.should_exit = True + if http_task is not None: + try: + await asyncio.wait_for(http_task, timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 + http_task.cancel() + await client.aclose() + return 0 + + +def run_headless(pair: Optional[tuple[str, int, str, str]] = None) -> int: + cfg = load_config() + save_config(cfg) + client = MasterClient(cfg) + + async def _main() -> int: + if pair is not None: + host, port, code, name = pair + try: + await pair_cli(cfg, client, host, port, code, name) + except MasterError as exc: + log.error(f"pairing failed: {exc}") + await client.aclose() + return 1 + return await run_headless_async(cfg, client) + + try: + return asyncio.run(_main()) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + sys.exit(run_headless()) diff --git a/javis-slave/src/javis_slave/http_server.py b/javis-slave/src/javis_slave/http_server.py new file mode 100644 index 0000000..82905be --- /dev/null +++ b/javis-slave/src/javis_slave/http_server.py @@ -0,0 +1,444 @@ +"""HTTP server for remote control of the slave device via MCP. + +Exposes endpoints for: +- Device status and info +- Audio control (mute, volume) +- Wake-word and room settings +- Audio engine state +""" +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Awaitable, Callable, Optional + +from fastapi import FastAPI, HTTPException, Query +from pydantic import BaseModel + +if TYPE_CHECKING: + from .audio_engine import AudioEngine # I-6: was .app (circular import risk) + from .config import SlaveConfig + +from .logging_setup import get_logger # I-7: route through the javis_slave root logger +from .system_control import ( + get_brightness_controller, + get_power_controller, + get_volume_controller, +) + +log = get_logger("http") + + +# ---- Models ---- + + +class SlaveStatus(BaseModel): + """Current status of the slave device.""" + device_id: str + device_name: str + room: Optional[str] = None + paired: bool + engine_running: bool + audio_state: str # "idle", "listening", "thinking", "speaking", "muted", "error" + muted: bool = False + mic_level: float = 0.0 + wake_word_score: float = 0.0 + last_error: Optional[str] = None + + +class DeviceInfo(BaseModel): + """Static device information.""" + device_id: str + device_name: str + sample_rate: int + input_device: Optional[int] = None + output_device: Optional[int] = None + wake_word_model: str + + +class AudioState(BaseModel): + """Audio system state.""" + mic_enabled: bool + output_enabled: bool + muted: bool = False + mic_level: float + wake_word_score: float + + +class RoomInfo(BaseModel): + """Room/location info.""" + room: Optional[str] = None + device_name: str + + +class ControlResponse(BaseModel): + """Response for control operations.""" + success: bool + message: str = "" + + +class VolumeResponse(BaseModel): + """Volume control response.""" + success: bool + message: str = "" + volume: Optional[float] = None # 0.0-1.0 + + +class BrightnessResponse(BaseModel): + """Brightness control response.""" + success: bool + message: str = "" + brightness: Optional[int] = None # 0-100 + + +class PowerResponse(BaseModel): + """Power control response.""" + success: bool + message: str = "" + + +class HealthResponse(BaseModel): + """Health check response.""" + ok: bool + device_id: str + engine_running: bool + + +class MasterSlaveInfo(BaseModel): + """A slave device as reported by the master (master-side auth).""" + device_id: str + name: str + room: Optional[str] = None + ip_address: Optional[str] = None + last_seen: Optional[str] = None + last_seen_ip: Optional[str] = None + created_at: Optional[str] = None + + +class MasterSlaveIp(BaseModel): + """IP info for a single slave, proxied from the master.""" + device_id: str + ip_address: Optional[str] = None + last_seen_ip: Optional[str] = None + last_seen: Optional[str] = None + + +# ---- Factory ---- + + +def build_http_app( + cfg: SlaveConfig, + engine_getter: Callable[[], Optional[AudioEngine]], + state_getter: Callable[[], str], + error_getter: Callable[[], Optional[str]], + mic_level_getter: Callable[[], tuple[float, float]], # (rms, score) + on_restart: Optional[Callable[[], Awaitable[None]]] = None, + master_client_getter: Optional[Callable[[], object]] = None, +) -> FastAPI: + """Create FastAPI app with slave control endpoints. + + Args: + cfg: Slave configuration (device_id, pairing info, etc.) + engine_getter: Callable that returns the current AudioEngine or None + state_getter: Callable that returns current audio state string + error_getter: Callable that returns last error message or None + mic_level_getter: Callable that returns (rms, wake_word_score) tuple + on_restart: Optional async callback invoked by ``POST /mcp/v1/restart`` + to perform a real stop+start of the audio engine. When omitted, + the endpoint returns a 501 to make the "not wired" state explicit + instead of a misleading 200. + master_client_getter: Optional callable returning a ``MasterClient`` + used by the ``/mcp/v1/master/*`` pass-through endpoints to query + the master with this slave's own credentials. When omitted, those + endpoints return 501. + """ + app = FastAPI(title="Javis Slave Control") + + # ---- Status Endpoints ---- + + @app.get("/mcp/v1/status", response_model=SlaveStatus) + async def get_status() -> SlaveStatus: + """Get current device status.""" + engine = engine_getter() + rms, score = mic_level_getter() + return SlaveStatus( + device_id=cfg.device_id, + device_name=cfg.device_name, + room=cfg.room, + paired=cfg.paired, + engine_running=engine is not None, + audio_state=state_getter(), + muted=bool(engine is not None and engine.muted), + mic_level=rms, + wake_word_score=score, + last_error=error_getter(), + ) + + @app.get("/mcp/v1/device-info", response_model=DeviceInfo) + async def get_device_info() -> DeviceInfo: + """Get static device information.""" + return DeviceInfo( + device_id=cfg.device_id, + device_name=cfg.device_name, + sample_rate=cfg.sample_rate, + input_device=cfg.input_device, + output_device=cfg.output_device, + wake_word_model=cfg.wake_word.model, + ) + + @app.get("/mcp/v1/room", response_model=RoomInfo) + async def get_room() -> RoomInfo: + """Get room/location information.""" + return RoomInfo(room=cfg.room, device_name=cfg.device_name) + + @app.get("/mcp/v1/audio-state", response_model=AudioState) + async def get_audio_state() -> AudioState: + """Get audio system state.""" + engine = engine_getter() + rms, score = mic_level_getter() + muted = bool(engine is not None and engine.muted) + return AudioState( + mic_enabled=engine is not None and not muted, + output_enabled=engine is not None, + muted=muted, + mic_level=rms, + wake_word_score=score, + ) + + # ---- Control Endpoints ---- + + @app.post("/mcp/v1/mic/mute", response_model=ControlResponse) + async def mute_mic() -> ControlResponse: + """Mute the microphone.""" + engine = engine_getter() + if engine is None or engine.mic is None: + raise HTTPException(status_code=400, detail="Engine not running") + try: + await engine.mute() + return ControlResponse(success=True, message="Microphone muted") + except Exception as e: + return ControlResponse(success=False, message=str(e)) + + @app.post("/mcp/v1/mic/unmute", response_model=ControlResponse) + async def unmute_mic() -> ControlResponse: + """Unmute the microphone.""" + engine = engine_getter() + if engine is None or engine.mic is None: + raise HTTPException(status_code=400, detail="Engine not running") + try: + await engine.unmute() + return ControlResponse(success=True, message="Microphone unmuted") + except Exception as e: + return ControlResponse(success=False, message=str(e)) + + @app.post("/mcp/v1/mic/toggle", response_model=ControlResponse) + async def toggle_mic() -> ControlResponse: + """Toggle microphone on/off.""" + engine = engine_getter() + if engine is None or engine.mic is None: + raise HTTPException(status_code=400, detail="Engine not running") + try: + now_muted = await engine.toggle_mute() + if now_muted: + return ControlResponse(success=True, message="Microphone muted") + return ControlResponse(success=True, message="Microphone unmuted") + except Exception as e: + return ControlResponse(success=False, message=str(e)) + + @app.post("/mcp/v1/restart", response_model=ControlResponse) + async def restart_audio_engine() -> ControlResponse: + """Restart the audio engine (for recovery from errors). + + Delegates to the app's ``on_restart`` callback (stop + start). If the + app did not wire a callback, return 501 so callers don't receive a + misleading 200 for a restart that will not happen. + """ + if on_restart is None: + raise HTTPException(status_code=501, detail="Engine restart not available") + log.info("Audio engine restart requested via HTTP") + try: + await on_restart() + return ControlResponse(success=True, message="Audio engine restarted") + except Exception as e: # noqa: BLE001 + return ControlResponse(success=False, message=str(e)) + + # ---- Master Pass-Through Endpoints ---- + # These proxy the master's /api/v1/slaves endpoints so an MCP client can + # query the master using THIS slave's own credentials. This removes the + # MCP's need to hardcode master host + auth (it used to call :8765 with + # no headers and always get 401). + + @app.get("/mcp/v1/master/slaves", response_model=list[MasterSlaveInfo]) + async def master_list_slaves() -> list[MasterSlaveInfo]: + """List all slave devices known to the master (authenticated pass-through).""" + if master_client_getter is None: + raise HTTPException(status_code=501, detail="Master query not available") + client = master_client_getter() + try: + data = await client.list_slaves() + except Exception as e: # noqa: BLE001 + raise HTTPException(status_code=502, detail=f"Master query failed: {e}") + return [MasterSlaveInfo(**item) for item in data] + + @app.get("/mcp/v1/master/slaves/{device_id}/ip", response_model=MasterSlaveIp) + async def master_slave_ip(device_id: str) -> MasterSlaveIp: + """Return the IP info for a specific slave device (authenticated pass-through).""" + if master_client_getter is None: + raise HTTPException(status_code=501, detail="Master query not available") + client = master_client_getter() + try: + data = await client.list_slaves() + except Exception as e: # noqa: BLE001 + raise HTTPException(status_code=502, detail=f"Master query failed: {e}") + for item in data: + if item.get("device_id") == device_id: + return MasterSlaveIp( + device_id=device_id, + ip_address=item.get("ip_address"), + last_seen_ip=item.get("last_seen_ip"), + last_seen=item.get("last_seen"), + ) + raise HTTPException(status_code=404, detail=f"Slave '{device_id}' not found on master") + + # ---- Volume Control Endpoints ---- + # Controllers shell out (pactl/amixer/systemctl on Linux) — run them off + # the event loop so a slow call can't stall wake-word processing. + + @app.get("/mcp/v1/volume", response_model=VolumeResponse) + async def get_volume() -> VolumeResponse: + """Get current master volume level (0.0-1.0).""" + volume = await asyncio.to_thread(get_volume_controller().get_volume) + if volume is not None: + return VolumeResponse(success=True, volume=volume, message=f"Volume: {volume * 100:.1f}%") + return VolumeResponse(success=False, message="Failed to get volume") + + @app.post("/mcp/v1/volume/set", response_model=VolumeResponse) + async def set_volume( + level: float = Query(..., ge=0.0, le=1.0), + ) -> VolumeResponse: + """Set master volume level (0.0-1.0).""" + if await asyncio.to_thread(get_volume_controller().set_volume, level): + return VolumeResponse(success=True, volume=level, message=f"Volume set to {level * 100:.1f}%") + return VolumeResponse(success=False, message="Failed to set volume") + + @app.post("/mcp/v1/volume/increase", response_model=VolumeResponse) + async def increase_volume( + amount: float = Query(0.05, ge=0.0, le=1.0), + ) -> VolumeResponse: + """Increase master volume by *amount* (0.0–1.0).""" + new_volume = await asyncio.to_thread(get_volume_controller().increase_volume, amount) + if new_volume is not None: + return VolumeResponse(success=True, volume=new_volume, message=f"Volume increased to {new_volume * 100:.1f}%") + return VolumeResponse(success=False, message="Failed to increase volume") + + @app.post("/mcp/v1/volume/decrease", response_model=VolumeResponse) + async def decrease_volume( + amount: float = Query(0.05, ge=0.0, le=1.0), + ) -> VolumeResponse: + """Decrease master volume by *amount* (0.0–1.0).""" + new_volume = await asyncio.to_thread(get_volume_controller().decrease_volume, amount) + if new_volume is not None: + return VolumeResponse(success=True, volume=new_volume, message=f"Volume decreased to {new_volume * 100:.1f}%") + return VolumeResponse(success=False, message="Failed to decrease volume") + + @app.post("/mcp/v1/volume/mute", response_model=VolumeResponse) + async def mute_volume() -> VolumeResponse: + """Mute master audio output.""" + if await asyncio.to_thread(get_volume_controller().set_mute, True): + return VolumeResponse(success=True, message="Audio muted") + return VolumeResponse(success=False, message="Failed to mute audio") + + @app.post("/mcp/v1/volume/unmute", response_model=VolumeResponse) + async def unmute_volume() -> VolumeResponse: + """Unmute master audio output.""" + if await asyncio.to_thread(get_volume_controller().set_mute, False): + return VolumeResponse(success=True, message="Audio unmuted") + return VolumeResponse(success=False, message="Failed to unmute audio") + + # ---- Brightness Control Endpoints ---- + + @app.get("/mcp/v1/brightness", response_model=BrightnessResponse) + async def get_brightness() -> BrightnessResponse: + """Get current display brightness (0-100).""" + brightness = await asyncio.to_thread(get_brightness_controller().get_brightness) + if brightness is not None: + return BrightnessResponse(success=True, brightness=brightness, message=f"Brightness: {brightness}%") + return BrightnessResponse(success=False, message="Failed to get brightness") + + @app.post("/mcp/v1/brightness/set", response_model=BrightnessResponse) + async def set_brightness( + level: int = Query(..., ge=0, le=100), + ) -> BrightnessResponse: + """Set display brightness (0-100).""" + if await asyncio.to_thread(get_brightness_controller().set_brightness, level): + return BrightnessResponse(success=True, brightness=level, message=f"Brightness set to {level}%") + return BrightnessResponse(success=False, message="Failed to set brightness") + + @app.post("/mcp/v1/brightness/increase", response_model=BrightnessResponse) + async def increase_brightness( + amount: int = Query(5, ge=0, le=100), + ) -> BrightnessResponse: + """Increase display brightness by *amount* percentage points.""" + new_brightness = await asyncio.to_thread(get_brightness_controller().increase_brightness, amount) + if new_brightness is not None: + return BrightnessResponse(success=True, brightness=new_brightness, message=f"Brightness increased to {new_brightness}%") + return BrightnessResponse(success=False, message="Failed to increase brightness") + + @app.post("/mcp/v1/brightness/decrease", response_model=BrightnessResponse) + async def decrease_brightness( + amount: int = Query(5, ge=0, le=100), + ) -> BrightnessResponse: + """Decrease display brightness by *amount* percentage points.""" + new_brightness = await asyncio.to_thread(get_brightness_controller().decrease_brightness, amount) + if new_brightness is not None: + return BrightnessResponse(success=True, brightness=new_brightness, message=f"Brightness decreased to {new_brightness}%") + return BrightnessResponse(success=False, message="Failed to decrease brightness") + + # ---- Power Control Endpoints ---- + + @app.post("/mcp/v1/power/sleep", response_model=PowerResponse) + async def sleep_system() -> PowerResponse: + """Put system to sleep.""" + if await asyncio.to_thread(get_power_controller().sleep): + return PowerResponse(success=True, message="Sleep command sent") + return PowerResponse(success=False, message="Failed to send sleep command") + + @app.post("/mcp/v1/power/shutdown", response_model=PowerResponse) + async def shutdown_system( + delay: int = Query(0, ge=0, le=3600), + ) -> PowerResponse: + """Shutdown system. *delay* is seconds (0 = immediate, max 1h).""" + if await asyncio.to_thread(get_power_controller().shutdown, delay): + if delay > 0: + return PowerResponse(success=True, message=f"System will shutdown in {delay} seconds") + return PowerResponse(success=True, message="Shutdown command sent") + return PowerResponse(success=False, message="Failed to send shutdown command") + + @app.post("/mcp/v1/power/restart", response_model=PowerResponse) + async def restart_system( + delay: int = Query(0, ge=0, le=3600), + ) -> PowerResponse: + """Restart system. *delay* is seconds (0 = immediate, max 1h).""" + if await asyncio.to_thread(get_power_controller().restart, delay): + if delay > 0: + return PowerResponse(success=True, message=f"System will restart in {delay} seconds") + return PowerResponse(success=True, message="Restart command sent") + return PowerResponse(success=False, message="Failed to send restart command") + + @app.post("/mcp/v1/power/cancel", response_model=PowerResponse) + async def cancel_shutdown() -> PowerResponse: + """Cancel pending shutdown or restart.""" + if await asyncio.to_thread(get_power_controller().cancel_shutdown): + return PowerResponse(success=True, message="Shutdown/restart cancelled") + return PowerResponse(success=False, message="No pending shutdown/restart to cancel") + + @app.get("/mcp/v1/health", response_model=HealthResponse) + async def health() -> HealthResponse: + """Health check endpoint.""" + engine = engine_getter() + return HealthResponse( + ok=True, + device_id=cfg.device_id, + engine_running=engine is not None, + ) + + return app diff --git a/javis-slave/src/javis_slave/logging_setup.py b/javis-slave/src/javis_slave/logging_setup.py new file mode 100644 index 0000000..1193710 --- /dev/null +++ b/javis-slave/src/javis_slave/logging_setup.py @@ -0,0 +1,46 @@ +"""Minimal stdlib logging helper (kept dependency-free from the javis package). + +Logs go to stderr and to a rotating file under ``/logs/slave.log`` +so tablet/Pi problems can be diagnosed after the fact. +""" +from __future__ import annotations + +import logging +import sys +from logging.handlers import RotatingFileHandler + +_configured = False +_FMT = logging.Formatter( + "%(asctime)s | %(levelname)-7s | %(name)s - %(message)s", datefmt="%H:%M:%S", +) + + +def _configure_once() -> None: + global _configured + if _configured: + return + root = logging.getLogger("javis_slave") + root.setLevel(logging.INFO) + root.propagate = False + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(_FMT) + root.addHandler(handler) + + try: + from .config import config_dir + log_dir = config_dir() / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + fh = RotatingFileHandler(log_dir / "slave.log", maxBytes=2_000_000, + backupCount=3, encoding="utf-8") + fh.setFormatter(logging.Formatter( + "%(asctime)s | %(levelname)-7s | %(name)s - %(message)s")) + root.addHandler(fh) + except Exception as exc: # noqa: BLE001 (read-only FS etc. — stderr still works) + root.warning(f"file logging disabled: {exc}") + _configured = True + + +def get_logger(name: str) -> logging.Logger: + _configure_once() + return logging.getLogger(f"javis_slave.{name}") diff --git a/javis-slave/src/javis_slave/main_window.py b/javis-slave/src/javis_slave/main_window.py new file mode 100644 index 0000000..fd49a81 --- /dev/null +++ b/javis-slave/src/javis_slave/main_window.py @@ -0,0 +1,584 @@ +"""Touch-optimized PySide6 UI for javis-slave (Surface Pro 7 kiosk style). + +Same VS Code Dark Modern palette as the Javis master GUI, with all touch +targets ≥ 48 px. +""" +from __future__ import annotations + +from typing import Callable, Optional + +from PySide6 import QtCore, QtGui, QtWidgets +from PySide6.QtCore import QTimer + +# ── palette (mirrors javis/gui/main_window.py) ──────────────────────────────── +CL_BG = "#1e1e1e" +CL_SURFACE = "#252526" +CL_SURFACE2 = "#2d2d2d" +CL_INPUT = "#3c3c3c" +CL_ACCENT = "#DA7756" +CL_ACCENT2 = "#e0886a" +CL_TEXT = "#cccccc" +CL_MUTED = "#858585" +CL_BORDER = "#3c3c3c" + +# Gain applied to raw mic RMS before driving the orb pulse (A-12: was magic 8.0). +# Typical speech RMS is ~0.01–0.1, so ×8 maps it into the visible 0..1 range. +_RMS_GAIN = 8.0 + +STATE_COLORS = { + "idle": "#3c3c3c", + "listening": "#4ec994", + "thinking": CL_ACCENT, + "speaking": "#4ec994", + "muted": "#f14c4c", + "error": "#f14c4c", +} + +STATE_LABELS = { + "idle": "Sag das Wake-Word …", + "listening": "Ich höre zu …", + "thinking": "Einen Moment …", + "speaking": "Antwort wird abgespielt …", + "muted": "Stumm geschaltet", + "error": "Fehler", +} + +_INPUT_QSS = ( + f"QLineEdit {{ background: {CL_INPUT}; color: {CL_TEXT}; border: 1px solid {CL_BORDER};" + f" border-radius: 12px; padding: 14px 18px; font-size: 16pt; }}" + f"QLineEdit:focus {{ border: 2px solid {CL_ACCENT}; }}" +) + +_PRIMARY_BTN_QSS = ( + f"QPushButton {{ background: {CL_ACCENT}; color: #1e1e1e; font-weight: 700;" + f" border: 0; border-radius: 14px; padding: 16px 32px; font-size: 16pt; }}" + f"QPushButton:pressed {{ background: {CL_ACCENT2}; }}" + f"QPushButton:disabled {{ background: {CL_INPUT}; color: {CL_MUTED}; }}" +) + +_GHOST_BTN_QSS = ( + f"QPushButton {{ background: transparent; border: 1px solid {CL_BORDER}; color: {CL_MUTED};" + f" padding: 12px 24px; border-radius: 12px; font-size: 13pt; }}" + f"QPushButton:pressed {{ background: {CL_SURFACE2}; color: {CL_TEXT}; }}" +) + + +class StateOrb(QtWidgets.QWidget): + """Large animated circle reflecting the agent state. + + Alexa-style visual feedback: + - Color indicates device state (idle/listening/thinking/speaking/error) + - Pulse ring shows mic activity level in real-time + - Mute overlay (red X) when microphone is disabled + - Retry indicator (spinning dots) during reconnection attempts + """ + + def __init__(self, parent=None): + super().__init__(parent) + self._state = "idle" + self._level = 0.0 # mic RMS 0..1 (pulse while idle/listening) + self._muted = False # explicit mute state overlay + self._reconnecting = False # showing retry indicator + self.setMinimumSize(260, 260) + + def set_state(self, state: str) -> None: + self._state = state + self.update() + + def set_level(self, level: float) -> None: + self._level = max(0.0, min(1.0, level)) + self.update() + + def set_muted(self, muted: bool) -> None: + """Set the mute overlay state (Alexa-style mute indicator).""" + self._muted = muted + self.update() + + def set_reconnecting(self, reconnecting: bool) -> None: + """Show/hide reconnection retry indicator.""" + self._reconnecting = reconnecting + self.update() + + def is_muted(self) -> bool: + return self._muted + + def paintEvent(self, event): # noqa: N802 + p = QtGui.QPainter(self) + p.setRenderHint(QtGui.QPainter.Antialiasing) + rect = self.rect() + size = min(rect.width(), rect.height()) + base_r = size * 0.32 + pulse = base_r * (1.0 + 0.35 * self._level) + center = rect.center() + + color = QtGui.QColor(STATE_COLORS.get(self._state, CL_INPUT)) + + # Outer pulse ring. + ring = QtGui.QColor(color) + ring.setAlpha(60) + p.setBrush(ring) + p.setPen(QtCore.Qt.NoPen) + p.drawEllipse(center, pulse, pulse) + + # Core. + p.setBrush(color) + p.drawEllipse(center, base_r, base_r) + + # Icon glyph — dark on bright cores, light on the dark idle core so it stays visible. + bright_core = self._state in ("listening", "thinking", "speaking", "muted", "error") + p.setPen(QtGui.QPen(QtGui.QColor("#1e1e1e" if bright_core else CL_TEXT), 3)) + font = p.font() + font.setPointSizeF(size * 0.11) + p.setFont(font) + glyph = {"idle": "🎤", "listening": "🎤", "thinking": "…", + "speaking": "🔊", "muted": "🔇", "error": "!"}.get(self._state, "") + p.drawText(QtCore.QRectF(rect), QtCore.Qt.AlignCenter, glyph) + + # Mute overlay (Alexa-style red X indicator). + if self._muted: + mute_r = base_r * 0.5 + mute_center = QtCore.QPointF(center.x() + size * 0.28, + center.y() - size * 0.28) + p.setPen(QtGui.QPen(QtGui.QColor("#f14c4c"), max(2, size // 80))) + p.drawLine( + QtCore.QPointF(mute_center.x() - mute_r, mute_center.y() - mute_r), + QtCore.QPointF(mute_center.x() + mute_r, mute_center.y() + mute_r)) + p.drawLine( + QtCore.QPointF(mute_center.x() + mute_r, mute_center.y() - mute_r), + QtCore.QPointF(mute_center.x() - mute_r, mute_center.y() + mute_r)) + + # Reconnecting indicator (pulsing outer ring). + if self._reconnecting: + p.setPen(QtGui.QPen(QtGui.QColor("#f59e0b"), max(2, size // 60))) + ring_r = base_r * 1.35 + p.drawEllipse(center, ring_r, ring_r) + + p.end() + + +class SetupScreen(QtWidgets.QWidget): + """First-run pairing: master address + pairing code.""" + + pairRequested = QtCore.Signal(str, int, str, str) # host, port, code, name + + def __init__(self, parent=None): + super().__init__(parent) + self.setStyleSheet(f"background: {CL_BG};") + outer = QtWidgets.QVBoxLayout(self) + outer.setAlignment(QtCore.Qt.AlignCenter) + + card = QtWidgets.QFrame() + card.setMaximumWidth(620) + card.setStyleSheet( + f"QFrame {{ background: {CL_SURFACE}; border: 1px solid {CL_BORDER};" + f" border-radius: 18px; }}") + v = QtWidgets.QVBoxLayout(card) + v.setContentsMargins(40, 36, 40, 36) + v.setSpacing(18) + + title = QtWidgets.QLabel("Javis Slave koppeln") + title.setStyleSheet( + f"color: {CL_TEXT}; font-size: 24pt; font-weight: 800; border: 0;") + title.setAlignment(QtCore.Qt.AlignCenter) + v.addWidget(title) + + hint = QtWidgets.QLabel( + "Erzeuge auf dem Javis-Master im Tab „Devices“ einen Pairing-Code\n" + "und trage hier die angezeigte Adresse und den Code ein.") + hint.setStyleSheet(f"color: {CL_MUTED}; font-size: 12pt; border: 0;") + hint.setAlignment(QtCore.Qt.AlignCenter) + v.addWidget(hint) + + self.host_edit = QtWidgets.QLineEdit() + self.host_edit.setPlaceholderText("Master-IP (z. B. 192.168.1.20)") + self.host_edit.setStyleSheet(_INPUT_QSS) + v.addWidget(self.host_edit) + + self.port_edit = QtWidgets.QLineEdit("8765") + self.port_edit.setPlaceholderText("Port") + self.port_edit.setValidator(QtGui.QIntValidator(1, 65535, self)) + self.port_edit.setStyleSheet(_INPUT_QSS) + v.addWidget(self.port_edit) + + self.code_edit = QtWidgets.QLineEdit() + self.code_edit.setPlaceholderText("Pairing-Code") + self.code_edit.setMaxLength(6) + self.code_edit.setStyleSheet(_INPUT_QSS + "QLineEdit { letter-spacing: 6px; }") + v.addWidget(self.code_edit) + + self.name_edit = QtWidgets.QLineEdit() + self.name_edit.setPlaceholderText("Gerätename (z. B. Tablet Küche)") + self.name_edit.setStyleSheet(_INPUT_QSS) + v.addWidget(self.name_edit) + + self.status_lbl = QtWidgets.QLabel("") + self.status_lbl.setStyleSheet("color: #f14c4c; font-size: 12pt; border: 0;") + self.status_lbl.setAlignment(QtCore.Qt.AlignCenter) + self.status_lbl.setWordWrap(True) + v.addWidget(self.status_lbl) + + self.pair_btn = QtWidgets.QPushButton("Koppeln") + self.pair_btn.setStyleSheet(_PRIMARY_BTN_QSS) + self.pair_btn.setCursor(QtCore.Qt.PointingHandCursor) + self.pair_btn.clicked.connect(self._submit) + v.addWidget(self.pair_btn) + + outer.addWidget(card) + + def _submit(self) -> None: + host = self.host_edit.text().strip() + port = int(self.port_edit.text() or "8765") + code = self.code_edit.text().strip() + name = self.name_edit.text().strip() + if not host or not code: + self.status_lbl.setText("Bitte Master-IP und Pairing-Code eingeben.") + return + self.status_lbl.setStyleSheet(f"color: {CL_MUTED}; font-size: 12pt; border: 0;") + self.status_lbl.setText("Verbinde …") + self.pair_btn.setEnabled(False) + self.pairRequested.emit(host, port, code, name) + + def show_error(self, message: str) -> None: + self.status_lbl.setStyleSheet("color: #f14c4c; font-size: 12pt; border: 0;") + self.status_lbl.setText(message) + self.pair_btn.setEnabled(True) + + +class MainScreen(QtWidgets.QWidget): + """Paired mode: state orb, transcript card, status badges. + + Alexa-style UX features: + - Mute toggle button on the orb + - Connection health monitoring with retry countdown + - Transcript history (scrollable conversation log) + - Visual mute indicator overlay + """ + + settingsRequested = QtCore.Signal() + muteToggled = QtCore.Signal(bool) # new signal for mute toggle + + def __init__(self, parent=None): + super().__init__(parent) + self.setStyleSheet(f"background: {CL_BG};") + outer = QtWidgets.QVBoxLayout(self) + outer.setContentsMargins(28, 20, 28, 24) + outer.setSpacing(16) + + # ── top bar: room badge · connection health · mute · settings ────── + top = QtWidgets.QHBoxLayout() + self.room_lbl = QtWidgets.QLabel("Raum: —") + self.room_lbl.setStyleSheet( + f"color: {CL_TEXT}; background: {CL_SURFACE2}; border-radius: 14px;" + f"padding: 10px 22px; font-size: 14pt; font-weight: 600;") + top.addWidget(self.room_lbl) + top.addStretch(1) + + # Connection status with retry countdown + self.conn_lbl = QtWidgets.QLabel("● verbunden") + self.conn_lbl.setStyleSheet( + "color: #4ec994; font-size: 13pt; padding: 10px;") + top.addWidget(self.conn_lbl) + + # Mute toggle button (Alexa-style red ring when muted) + self.mute_btn = QtWidgets.QPushButton("🎤") + self.mute_btn.setFixedSize(56, 56) + self.mute_btn.setCursor(QtCore.Qt.PointingHandCursor) + self.mute_btn.setStyleSheet( + f"QPushButton {{ background: {CL_SURFACE2}; color: {CL_TEXT}; border: 0;" + f" border-radius: 14px; font-size: 18pt; }}" + f"QPushButton:checked {{ background: #f14c4c; color: white; }}" + f"QPushButton:pressed {{ background: {CL_INPUT}; color: {CL_TEXT}; }}" + f"QPushButton:disabled {{ background: {CL_SURFACE}; color: {CL_MUTED}; }}") + self.mute_btn.setCheckable(True) # Alexa-style toggle state + self.mute_btn.clicked.connect(self._on_mute_toggled) + top.addWidget(self.mute_btn) + + gear = QtWidgets.QPushButton("⚙") + gear.setFixedSize(56, 56) + gear.setCursor(QtCore.Qt.PointingHandCursor) + gear.setStyleSheet( + f"QPushButton {{ background: {CL_SURFACE2}; color: {CL_MUTED}; border: 0;" + f" border-radius: 14px; font-size: 18pt; }}" + f"QPushButton:pressed {{ background: {CL_INPUT}; color: {CL_TEXT}; }}") + gear.clicked.connect(self.settingsRequested.emit) + top.addWidget(gear) + outer.addLayout(top) + + # Retry countdown label (hidden by default, shown during reconnection) + self.retry_lbl = QtWidgets.QLabel("") + self.retry_lbl.setStyleSheet( + f"color: #f59e0b; font-size: 12pt; font-weight: 600; border: 0;") + self.retry_lbl.setAlignment(QtCore.Qt.AlignCenter) + self.retry_lbl.setVisible(False) + outer.addWidget(self.retry_lbl) + + # ── state orb + label ───────────────────────────────────────────── + self.orb = StateOrb() + outer.addWidget(self.orb, 2) + self.state_lbl = QtWidgets.QLabel(STATE_LABELS["idle"]) + self.state_lbl.setAlignment(QtCore.Qt.AlignCenter) + self.state_lbl.setStyleSheet(f"color: {CL_MUTED}; font-size: 16pt;") + outer.addWidget(self.state_lbl) + + # ── transcript card (scrollable conversation history) ────────────── + card = QtWidgets.QFrame() + card.setStyleSheet( + f"QFrame {{ background: {CL_SURFACE}; border: 1px solid {CL_BORDER};" + f" border-radius: 18px; }}") + cv = QtWidgets.QVBoxLayout(card) + cv.setContentsMargins(28, 22, 28, 22) + cv.setSpacing(10) + + # Header label + history_header = QtWidgets.QLabel("Konversation") + history_header.setStyleSheet(f"color: {CL_MUTED}; font-size: 11pt; border: 0;") + cv.addWidget(history_header) + + # Transcript history (replaces single-turn display). + # NOTE: this layout must belong to *exactly one* parent — we set it on + # ``transcript_container`` (inside the scroll area) below, so it must + # NOT also be added to ``cv`` (Qt raises RuntimeError: "Attempting to + # set QLayout ... when the QLayout already has a parent"). + self.transcript_layout = QtWidgets.QVBoxLayout() + self.transcript_layout.setSpacing(8) + + # Scroll area for transcript + self.transcript_scroll = QtWidgets.QScrollArea() + self.transcript_scroll.setWidgetResizable(True) + self.transcript_scroll.setFrameShape(QtWidgets.QFrame.NoFrame) + self.transcript_scroll.setStyleSheet("background: transparent; border: 0;") + self.transcript_container = QtWidgets.QWidget() + self.transcript_container.setLayout(self.transcript_layout) + self.transcript_scroll.setWidget(self.transcript_container) + QtWidgets.QScroller.grabGesture( + self.transcript_scroll.viewport(), QtWidgets.QScroller.TouchGesture) + cv.addWidget(self.transcript_scroll, 1) + + outer.addWidget(card, 3) + + # Track last state for change detection + self._last_state = "idle" + + def _on_mute_toggled(self) -> None: + """Handle mute button toggle.""" + is_muted = self.mute_btn.isChecked() + self.muteToggled.emit(is_muted) + + # ── slots (called via Qt signals from the engine bridge) ────────────── + + def set_state(self, state: str) -> None: + self.orb.set_state(state) + self.orb.set_muted(state == "muted") # sync mute overlay with state + label = STATE_LABELS.get(state, state) + if label: + self.state_lbl.setText(label) + # Mute toggle stays in place (enabled/disabled) so the top bar never + # jumps while the state cycles; it is only inactive mid-turn. + interactive_states = ("idle", "listening", "speaking", "muted") + self.mute_btn.setEnabled(state in interactive_states) + # Sync the button's checked state to match the engine's mute state. + want_checked = (state == "muted") + if self.mute_btn.isChecked() != want_checked: + self.mute_btn.blockSignals(True) + self.mute_btn.setChecked(want_checked) + self.mute_btn.blockSignals(False) + + def set_mic_level(self, rms: float, score: float) -> None: + self.orb.set_level(min(1.0, rms * _RMS_GAIN)) + + def _ensure_transcript_spacer(self) -> None: + """Remove existing spacer (if any) and add a fresh one at the end.""" + # Walk backwards to find and remove any existing spacer items + while self.transcript_layout.count() > 0: + item = self.transcript_layout.itemAt(self.transcript_layout.count() - 1) + if isinstance(item, QtWidgets.QSpacerItem): + self.transcript_layout.removeItem(item) + else: + break + spacer = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, + QtWidgets.QSizePolicy.Expanding) + self.transcript_layout.addSpacerItem(spacer) + + def _prune_transcript(self, max_bubbles: int = 40) -> None: + """Keep at most *max_bubbles* bubbles in the transcript layout (oldest first).""" + count = self.transcript_layout.count() + bubbles = count + if count > 0 and isinstance(self.transcript_layout.itemAt(count - 1), QtWidgets.QSpacerItem): + bubbles -= 1 + excess = bubbles - max_bubbles + for _ in range(max(0, excess)): + item = self.transcript_layout.takeAt(0) + if item is not None and item.widget(): + item.widget().deleteLater() + + def set_turn(self, user_text: str, reply_text: str) -> None: + """Add a new conversation turn to the transcript history (Alexa-style).""" + # User message bubble + user_bubble = QtWidgets.QLabel(f"👤 {user_text}") + user_bubble.setStyleSheet( + f"background: {CL_SURFACE2}; color: {CL_TEXT}; border-radius: 12px;" + f"padding: 12px 18px; font-size: 14pt;") + user_bubble.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) + user_bubble.setWordWrap(True) + self.transcript_layout.addWidget(user_bubble) + + # Assistant message bubble (if any) + if reply_text: + reply_bubble = QtWidgets.QLabel(f"🤖 {reply_text}") + reply_bubble.setStyleSheet( + f"background: {CL_ACCENT}; color: #1e1e1e; border-radius: 12px;" + f"padding: 12px 18px; font-size: 14pt; font-weight: 500;") + reply_bubble.setAlignment(QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter) + reply_bubble.setWordWrap(True) + self.transcript_layout.addWidget(reply_bubble) + + self._prune_transcript() + # Spacer to push content to bottom (like Alexa's conversation history) + self._ensure_transcript_spacer() + + # Auto-scroll to bottom + QTimer.singleShot(50, self._scroll_transcript_to_bottom) + + def _scroll_transcript_to_bottom(self) -> None: + """Auto-scroll transcript to show latest messages.""" + vbar = self.transcript_scroll.verticalScrollBar() + vbar.setValue(vbar.maximum()) + + def set_connection(self, ok: bool, retry_count: int = 0, backoff: float = 0.0) -> None: + if ok: + self.conn_lbl.setText("● verbunden") + self.conn_lbl.setStyleSheet("color: #4ec994; font-size: 13pt; padding: 10px;") + self.retry_lbl.setVisible(False) + self.orb.set_reconnecting(False) + else: + if retry_count > 0 and backoff > 0: + # Show retry countdown (Alexa-style progressive feedback) + text = f"○ trennung · versuch {retry_count} in {backoff:.0f}s" + self.conn_lbl.setText(text) + self.conn_lbl.setStyleSheet("color: #f59e0b; font-size: 13pt; padding: 10px;") + self.retry_lbl.setVisible(True) + self.retry_lbl.setText(f"Wiederverbindungsversuch in {backoff:.0f}s …") + self.orb.set_reconnecting(True) + else: + self.conn_lbl.setText("● getrennt") + self.conn_lbl.setStyleSheet("color: #f14c4c; font-size: 13pt; padding: 10px;") + + def set_room(self, room: Optional[str]) -> None: + self.room_lbl.setText(f"Raum: {room}" if room else "Raum: —") + + def show_error(self, message: str) -> None: + """Show a recoverable error to the user (Alexa-style progressive disclosure).""" + self._prune_transcript() + error_bubble = QtWidgets.QLabel(f"⚠ {message}") + error_bubble.setStyleSheet( + f"color: #f14c4c; font-size: 14pt; border: 0; padding: 12px;") + error_bubble.setWordWrap(True) + self.transcript_layout.addWidget(error_bubble) + self._ensure_transcript_spacer() + QTimer.singleShot(50, self._scroll_transcript_to_bottom) + + +class SlaveWindow(QtWidgets.QMainWindow): + """Top-level window switching between setup and main screens. + + Wired to engine callbacks that bridge asyncio events to Qt signals: + - engineState: device state changes (idle/listening/thinking/speaking/error) + - engineMicLevel: real-time audio level for orb pulse animation + - engineTurn: conversation turn results (user text + assistant reply) + - engineError: error messages from the engine + - engineConnection: connection health updates with retry countdown + - engineRoom: room assignment changes + """ + + # Bridged engine events (emitted from asyncio, consumed on the Qt side). + engineState = QtCore.Signal(str) + engineMicLevel = QtCore.Signal(float, float) + engineTurn = QtCore.Signal(str, str) + engineError = QtCore.Signal(str) + engineConnection = QtCore.Signal(bool, int, float) # (ok, retry_count, backoff) + engineRoom = QtCore.Signal(object) + + def __init__(self): + super().__init__() + self.setWindowTitle("Javis Slave") + self.resize(1368, 912) # Surface Pro 7 logical resolution @150 % + self.setStyleSheet(f"background: {CL_BG};") + + self.stack = QtWidgets.QStackedWidget() + self.setup_screen = SetupScreen() + self.main_screen = MainScreen() + self.stack.addWidget(self.setup_screen) + self.stack.addWidget(self.main_screen) + self.setCentralWidget(self.stack) + + self.engineState.connect(self.main_screen.set_state) + self.engineMicLevel.connect(self.main_screen.set_mic_level) + self.engineTurn.connect(self.main_screen.set_turn) + self.engineError.connect(self.main_screen.show_error) + self.engineConnection.connect(self.main_screen.set_connection) + self.engineRoom.connect(lambda r: self.main_screen.set_room(r)) + + QtGui.QShortcut(QtGui.QKeySequence("F11"), self, activated=self._toggle_fullscreen) + # Kiosk: fullscreen with a hidden cursor (touch panels); Esc leaves fullscreen. + self._kiosk = False + QtGui.QShortcut(QtGui.QKeySequence("Esc"), self, activated=self._leave_fullscreen) + + def set_kiosk(self, enabled: bool) -> None: + self._kiosk = enabled + if enabled: + self.showFullScreen() + QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CursorShape.BlankCursor) + + def _leave_fullscreen(self) -> None: + if self.isFullScreen(): + self.showNormal() + if self._kiosk: + QtWidgets.QApplication.restoreOverrideCursor() + + def show_setup(self) -> None: + self.stack.setCurrentWidget(self.setup_screen) + + def show_main(self) -> None: + self.stack.setCurrentWidget(self.main_screen) + + def _toggle_fullscreen(self) -> None: + if self.isFullScreen(): + self.showNormal() + else: + self.showFullScreen() + + def show_settings_dialog(self, device_id: str, device_name: str, + on_repair: Callable[[], None]) -> None: + dlg = QtWidgets.QDialog(self) + dlg.setWindowTitle("Einstellungen") + dlg.setStyleSheet(f"background: {CL_SURFACE}; color: {CL_TEXT};") + v = QtWidgets.QVBoxLayout(dlg) + v.setContentsMargins(28, 24, 28, 24) + v.setSpacing(14) + info = QtWidgets.QLabel( + f"Gerät: {device_name or '—'}\nGeräte-ID: {device_id}") + info.setStyleSheet(f"color: {CL_MUTED}; font-size: 12pt;") + info.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse) + v.addWidget(info) + btn_fs = QtWidgets.QPushButton("Vollbild umschalten (F11)") + btn_fs.setStyleSheet(_GHOST_BTN_QSS) + btn_fs.clicked.connect(lambda: (self._toggle_fullscreen(), dlg.accept())) + v.addWidget(btn_fs) + btn_repair = QtWidgets.QPushButton("Neu koppeln …") + btn_repair.setStyleSheet(_GHOST_BTN_QSS) + + def _do_repair(): + ret = QtWidgets.QMessageBox.question( + dlg, "Neu koppeln", + "Verbindung zum Master trennen und neu koppeln?") + if ret == QtWidgets.QMessageBox.Yes: + dlg.accept() + on_repair() + + btn_repair.clicked.connect(_do_repair) + v.addWidget(btn_repair) + btn_close = QtWidgets.QPushButton("Schließen") + btn_close.setStyleSheet(_PRIMARY_BTN_QSS) + btn_close.clicked.connect(dlg.accept) + v.addWidget(btn_close) + dlg.exec() diff --git a/javis-slave/src/javis_slave/system_control.py b/javis-slave/src/javis_slave/system_control.py new file mode 100644 index 0000000..a065f8b --- /dev/null +++ b/javis-slave/src/javis_slave/system_control.py @@ -0,0 +1,558 @@ +"""System control module for Javis Slave (volume, brightness, power). + +Windows: pycaw for audio, screen_brightness_control for display, ``shutdown`` +for power. Linux / Raspberry Pi: ``pactl`` (PipeWire/PulseAudio) or ``amixer`` +(ALSA) for audio, ``/sys/class/backlight`` for display, ``systemctl`` for power. +The HTTP layer only uses the ``get_*_controller()`` factories, which pick the +backend for the current platform. +""" + +import platform +import shutil +import subprocess +from pathlib import Path +from typing import Optional + +from .logging_setup import get_logger + +logger = get_logger("system") + +_IS_LINUX = platform.system() == "Linux" + + +class WindowsControlError(Exception): + """Exception for Windows control operations.""" + pass + + +class VolumeController: + """Control master volume on Windows.""" + + def __init__(self): + """Initialize volume controller.""" + self._pycaw_available = False + self._device = None + self._interface = None + + if platform.system() != "Windows": + logger.warning("Volume control only available on Windows") + return + + try: + from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume + + devices = AudioUtilities.GetSpeakers() + self._device = devices.Activate( + IAudioEndpointVolume._iid_, 0, None + ) + self._interface = self._device + self._pycaw_available = True + logger.info("Pycaw audio control initialized") + except Exception as e: + logger.warning(f"Failed to initialize pycaw: {e}") + + def get_volume(self) -> Optional[float]: + """Get current master volume (0.0 to 1.0). + + Returns: + Volume level 0.0-1.0, or None if unavailable. + """ + if not self._pycaw_available or not self._interface: + return None + + try: + return self._interface.GetMasterVolumeLevelScalar() + except Exception as e: + logger.error(f"Failed to get volume: {e}") + return None + + def set_volume(self, level: float) -> bool: + """Set master volume to level (0.0 to 1.0). + + Args: + level: Volume level 0.0-1.0. + + Returns: + True if successful, False otherwise. + """ + if not self._pycaw_available or not self._interface: + return False + + level = max(0.0, min(1.0, level)) + + try: + self._interface.SetMasterVolumeLevelScalar(level, None) + logger.info(f"Set volume to {level * 100:.1f}%") + return True + except Exception as e: + logger.error(f"Failed to set volume: {e}") + return False + + def is_muted(self) -> Optional[bool]: + """Check if audio is muted. + + Returns: + True if muted, False if not, None if unavailable. + """ + if not self._pycaw_available or not self._interface: + return None + + try: + return self._interface.GetMute() + except Exception as e: + logger.error(f"Failed to get mute state: {e}") + return None + + def set_mute(self, muted: bool) -> bool: + """Set mute state. + + Args: + muted: True to mute, False to unmute. + + Returns: + True if successful, False otherwise. + """ + if not self._pycaw_available or not self._interface: + return False + + try: + self._interface.SetMute(muted, None) + state = "muted" if muted else "unmuted" + logger.info(f"Audio {state}") + return True + except Exception as e: + logger.error(f"Failed to set mute: {e}") + return False + + def increase_volume(self, amount: float = 0.05) -> Optional[float]: + """Increase volume by amount. + + Args: + amount: Amount to increase (0.0-1.0). + + Returns: + New volume level, or None if failed. + """ + current = self.get_volume() + if current is None: + return None + + new_level = min(1.0, current + amount) + if self.set_volume(new_level): + return new_level + return None + + def decrease_volume(self, amount: float = 0.05) -> Optional[float]: + """Decrease volume by amount. + + Args: + amount: Amount to decrease (0.0-1.0). + + Returns: + New volume level, or None if failed. + """ + current = self.get_volume() + if current is None: + return None + + new_level = max(0.0, current - amount) + if self.set_volume(new_level): + return new_level + return None + + +class BrightnessController: + """Control display brightness on Windows.""" + + def __init__(self): + """Initialize brightness controller.""" + self._available = False + + if platform.system() != "Windows": + logger.warning("Brightness control only available on Windows") + return + + try: + import screen_brightness_control + self._brightness = screen_brightness_control + self._available = True + logger.info("Screen brightness control initialized") + except Exception as e: + logger.warning(f"Failed to initialize brightness control: {e}") + + def get_brightness(self) -> Optional[int]: + """Get current display brightness (0-100). + + Returns: + Brightness percentage 0-100, or None if unavailable. + """ + if not self._available: + return None + + try: + brightness = self._brightness.get_brightness() + # get_brightness() returns a list if multiple displays + if isinstance(brightness, list): + return int(brightness[0]) + return int(brightness) + except Exception as e: + logger.error(f"Failed to get brightness: {e}") + return None + + def set_brightness(self, level: int) -> bool: + """Set display brightness (0-100). + + Args: + level: Brightness percentage 0-100. + + Returns: + True if successful, False otherwise. + """ + if not self._available: + return False + + level = max(0, min(100, level)) + + try: + self._brightness.set_brightness(level) + logger.info(f"Set brightness to {level}%") + return True + except Exception as e: + logger.error(f"Failed to set brightness: {e}") + return False + + def increase_brightness(self, amount: int = 5) -> Optional[int]: + """Increase brightness by amount. + + Args: + amount: Amount to increase (percentage points). + + Returns: + New brightness level, or None if failed. + """ + current = self.get_brightness() + if current is None: + return None + + new_level = min(100, current + amount) + if self.set_brightness(new_level): + return new_level + return None + + def decrease_brightness(self, amount: int = 5) -> Optional[int]: + """Decrease brightness by amount. + + Args: + amount: Amount to decrease (percentage points). + + Returns: + New brightness level, or None if failed. + """ + current = self.get_brightness() + if current is None: + return None + + new_level = max(0, current - amount) + if self.set_brightness(new_level): + return new_level + return None + + +class PowerController: + """Control power settings on Windows.""" + + @staticmethod + def sleep() -> bool: + """Put system to sleep. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + subprocess.run(["rundll32.exe", "powrprof.dll,SetSuspendState", "0", "1", "0"], check=False) + logger.info("Sent sleep command") + return True + except Exception as e: + logger.error(f"Failed to sleep: {e}") + return False + + @staticmethod + def shutdown(delay: int = 0) -> bool: + """Shutdown system. + + Args: + delay: Delay in seconds before shutdown. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + if delay > 0: + subprocess.run(["shutdown", "/s", "/t", str(delay)], check=False) + logger.info(f"Scheduled shutdown in {delay} seconds") + else: + subprocess.run(["shutdown", "/s", "/t", "0"], check=False) + logger.info("Sent shutdown command") + return True + except Exception as e: + logger.error(f"Failed to shutdown: {e}") + return False + + @staticmethod + def restart(delay: int = 0) -> bool: + """Restart system. + + Args: + delay: Delay in seconds before restart. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + if delay > 0: + subprocess.run(["shutdown", "/r", "/t", str(delay)], check=False) + logger.info(f"Scheduled restart in {delay} seconds") + else: + subprocess.run(["shutdown", "/r", "/t", "0"], check=False) + logger.info("Sent restart command") + return True + except Exception as e: + logger.error(f"Failed to restart: {e}") + return False + + @staticmethod + def cancel_shutdown() -> bool: + """Cancel pending shutdown/restart. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + subprocess.run(["shutdown", "/a"], check=False) + logger.info("Cancelled pending shutdown/restart") + return True + except Exception as e: + logger.error(f"Failed to cancel shutdown: {e}") + return False + + +# ── Linux / Raspberry Pi backends ──────────────────────────────────────────── + +def _run(cmd: list[str], timeout: float = 5.0) -> Optional[str]: + """Run *cmd*; return stdout on success, None on any failure.""" + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.error(f"{cmd[0]} failed: {exc}") + return None + if proc.returncode != 0: + logger.error(f"{' '.join(cmd)} exited {proc.returncode}: {proc.stderr.strip()}") + return None + return proc.stdout + + +class LinuxVolumeController: + """Master volume via ``pactl`` (PipeWire/PulseAudio) with ``amixer`` (ALSA) fallback.""" + + def __init__(self, mixer_control: str = "Master"): + self._pactl = shutil.which("pactl") + self._amixer = shutil.which("amixer") + self._control = mixer_control + if not (self._pactl or self._amixer): + logger.warning("Neither pactl nor amixer found — volume control unavailable") + else: + logger.info(f"Linux volume control via {'pactl' if self._pactl else 'amixer'}") + + def _amixer_control(self) -> str: + # Pi HDMI/headphone cards expose "PCM" or "Headphone" instead of "Master". + out = _run([self._amixer, "scontrols"]) or "" + names = [line.split("'")[1] for line in out.splitlines() if "'" in line] + for cand in (self._control, "Master", "PCM", "Headphone", "Speaker", "Digital"): + if cand in names: + return cand + return names[0] if names else self._control + + def get_volume(self) -> Optional[float]: + if self._pactl: + out = _run([self._pactl, "get-sink-volume", "@DEFAULT_SINK@"]) + if out: + for tok in out.replace("/", " ").split(): + if tok.endswith("%") and tok[:-1].isdigit(): + return min(1.0, int(tok[:-1]) / 100.0) + if self._amixer: + out = _run([self._amixer, "get", self._amixer_control()]) + if out: + import re + m = re.search(r"\[(\d+)%\]", out) + if m: + return int(m.group(1)) / 100.0 + return None + + def set_volume(self, level: float) -> bool: + level = max(0.0, min(1.0, level)) + pct = f"{int(round(level * 100))}%" + if self._pactl and _run([self._pactl, "set-sink-volume", "@DEFAULT_SINK@", pct]) is not None: + logger.info(f"Set volume to {pct}") + return True + if self._amixer and _run([self._amixer, "set", self._amixer_control(), pct]) is not None: + logger.info(f"Set volume to {pct}") + return True + return False + + def is_muted(self) -> Optional[bool]: + if self._pactl: + out = _run([self._pactl, "get-sink-mute", "@DEFAULT_SINK@"]) + if out: + return "yes" in out.lower() + if self._amixer: + out = _run([self._amixer, "get", self._amixer_control()]) + if out: + return "[off]" in out + return None + + def set_mute(self, muted: bool) -> bool: + if self._pactl and _run([self._pactl, "set-sink-mute", "@DEFAULT_SINK@", "1" if muted else "0"]) is not None: + return True + if self._amixer and _run([self._amixer, "set", self._amixer_control(), "mute" if muted else "unmute"]) is not None: + return True + return False + + def increase_volume(self, amount: float = 0.05) -> Optional[float]: + cur = self.get_volume() + if cur is None: + return None + new = min(1.0, cur + amount) + return new if self.set_volume(new) else None + + def decrease_volume(self, amount: float = 0.05) -> Optional[float]: + cur = self.get_volume() + if cur is None: + return None + new = max(0.0, cur - amount) + return new if self.set_volume(new) else None + + +class LinuxBrightnessController: + """Backlight via ``/sys/class/backlight/*`` (official Pi touch display, DSI/eDP panels). + + Writing needs permission: add the user to the ``video`` group and install + the udev rule from RASPBERRY_PI.md, or run as root. + """ + + def __init__(self): + self._dev: Optional[Path] = None + for cand in sorted(Path("/sys/class/backlight").glob("*")): + if (cand / "brightness").exists() and (cand / "max_brightness").exists(): + self._dev = cand + break + if self._dev is None: + logger.warning("No /sys/class/backlight device — brightness control unavailable") + else: + logger.info(f"Linux brightness control via {self._dev}") + + def _max(self) -> int: + return max(1, int((self._dev / "max_brightness").read_text().strip())) # type: ignore[operator] + + def get_brightness(self) -> Optional[int]: + if self._dev is None: + return None + try: + cur = int((self._dev / "brightness").read_text().strip()) + return int(round(cur * 100 / self._max())) + except (OSError, ValueError) as exc: + logger.error(f"Failed to read brightness: {exc}") + return None + + def set_brightness(self, level: int) -> bool: + if self._dev is None: + return False + level = max(0, min(100, level)) + raw = int(round(level * self._max() / 100)) + try: + (self._dev / "brightness").write_text(str(raw)) + logger.info(f"Set brightness to {level}%") + return True + except OSError as exc: + logger.error(f"Failed to set brightness (permissions? see RASPBERRY_PI.md): {exc}") + return False + + def increase_brightness(self, amount: int = 5) -> Optional[int]: + cur = self.get_brightness() + if cur is None: + return None + new = min(100, cur + amount) + return new if self.set_brightness(new) else None + + def decrease_brightness(self, amount: int = 5) -> Optional[int]: + cur = self.get_brightness() + if cur is None: + return None + new = max(0, cur - amount) + return new if self.set_brightness(new) else None + + +class LinuxPowerController: + """Power via ``systemctl`` / ``shutdown`` (needs polkit rights or passwordless sudo — see RASPBERRY_PI.md).""" + + @staticmethod + def _priv(cmd: list[str]) -> list[str]: + import os + if os.geteuid() == 0 or not shutil.which("sudo"): + return cmd + return ["sudo", "-n", *cmd] + + def sleep(self) -> bool: + return _run(self._priv(["systemctl", "suspend"])) is not None + + def shutdown(self, delay: int = 0) -> bool: + when = f"+{max(1, (delay + 59) // 60)}" if delay > 0 else "now" + return _run(self._priv(["shutdown", "-h", when])) is not None + + def restart(self, delay: int = 0) -> bool: + when = f"+{max(1, (delay + 59) // 60)}" if delay > 0 else "now" + return _run(self._priv(["shutdown", "-r", when])) is not None + + def cancel_shutdown(self) -> bool: + return _run(self._priv(["shutdown", "-c"])) is not None + + +# Global controllers — one backend per platform. +if _IS_LINUX: + _volume = LinuxVolumeController() + _brightness = LinuxBrightnessController() + _power = LinuxPowerController() +else: + _volume = VolumeController() + _brightness = BrightnessController() + _power = PowerController() + + +def get_volume_controller() -> VolumeController: + """Get global volume controller.""" + return _volume + + +def get_brightness_controller() -> BrightnessController: + """Get global brightness controller.""" + return _brightness + + +def get_power_controller() -> PowerController: + """Get global power controller.""" + return _power diff --git a/javis-slave/src/javis_slave/windows_control.py b/javis-slave/src/javis_slave/windows_control.py new file mode 100644 index 0000000..bcfd06a --- /dev/null +++ b/javis-slave/src/javis_slave/windows_control.py @@ -0,0 +1,558 @@ +"""System control module for Javis Slave (volume, brightness, power). + +Windows: pycaw for audio, screen_brightness_control for display, ``shutdown`` +for power. Linux / Raspberry Pi: ``pactl`` (PipeWire/PulseAudio) or ``amixer`` +(ALSA) for audio, ``/sys/class/backlight`` for display, ``systemctl`` for power. +The HTTP layer only uses the ``get_*_controller()`` factories, which pick the +backend for the current platform. +""" + +import logging +import platform +import shutil +import subprocess +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +_IS_WINDOWS = platform.system() == "Windows" +_IS_LINUX = platform.system() == "Linux" + + +class WindowsControlError(Exception): + """Exception for Windows control operations.""" + pass + + +class VolumeController: + """Control master volume on Windows.""" + + def __init__(self): + """Initialize volume controller.""" + self._pycaw_available = False + self._device = None + self._interface = None + + if platform.system() != "Windows": + logger.warning("Volume control only available on Windows") + return + + try: + from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume + + devices = AudioUtilities.GetSpeakers() + self._device = devices.Activate( + IAudioEndpointVolume._iid_, 0, None + ) + self._interface = self._device + self._pycaw_available = True + logger.info("Pycaw audio control initialized") + except Exception as e: + logger.warning(f"Failed to initialize pycaw: {e}") + + def get_volume(self) -> Optional[float]: + """Get current master volume (0.0 to 1.0). + + Returns: + Volume level 0.0-1.0, or None if unavailable. + """ + if not self._pycaw_available or not self._interface: + return None + + try: + return self._interface.GetMasterVolumeLevelScalar() + except Exception as e: + logger.error(f"Failed to get volume: {e}") + return None + + def set_volume(self, level: float) -> bool: + """Set master volume to level (0.0 to 1.0). + + Args: + level: Volume level 0.0-1.0. + + Returns: + True if successful, False otherwise. + """ + if not self._pycaw_available or not self._interface: + return False + + level = max(0.0, min(1.0, level)) + + try: + self._interface.SetMasterVolumeLevelScalar(level, None) + logger.info(f"Set volume to {level * 100:.1f}%") + return True + except Exception as e: + logger.error(f"Failed to set volume: {e}") + return False + + def is_muted(self) -> Optional[bool]: + """Check if audio is muted. + + Returns: + True if muted, False if not, None if unavailable. + """ + if not self._pycaw_available or not self._interface: + return None + + try: + return self._interface.GetMute() + except Exception as e: + logger.error(f"Failed to get mute state: {e}") + return None + + def set_mute(self, muted: bool) -> bool: + """Set mute state. + + Args: + muted: True to mute, False to unmute. + + Returns: + True if successful, False otherwise. + """ + if not self._pycaw_available or not self._interface: + return False + + try: + self._interface.SetMute(muted, None) + state = "muted" if muted else "unmuted" + logger.info(f"Audio {state}") + return True + except Exception as e: + logger.error(f"Failed to set mute: {e}") + return False + + def increase_volume(self, amount: float = 0.05) -> Optional[float]: + """Increase volume by amount. + + Args: + amount: Amount to increase (0.0-1.0). + + Returns: + New volume level, or None if failed. + """ + current = self.get_volume() + if current is None: + return None + + new_level = min(1.0, current + amount) + if self.set_volume(new_level): + return new_level + return None + + def decrease_volume(self, amount: float = 0.05) -> Optional[float]: + """Decrease volume by amount. + + Args: + amount: Amount to decrease (0.0-1.0). + + Returns: + New volume level, or None if failed. + """ + current = self.get_volume() + if current is None: + return None + + new_level = max(0.0, current - amount) + if self.set_volume(new_level): + return new_level + return None + + +class BrightnessController: + """Control display brightness on Windows.""" + + def __init__(self): + """Initialize brightness controller.""" + self._available = False + + if platform.system() != "Windows": + logger.warning("Brightness control only available on Windows") + return + + try: + import screen_brightness_control + self._brightness = screen_brightness_control + self._available = True + logger.info("Screen brightness control initialized") + except Exception as e: + logger.warning(f"Failed to initialize brightness control: {e}") + + def get_brightness(self) -> Optional[int]: + """Get current display brightness (0-100). + + Returns: + Brightness percentage 0-100, or None if unavailable. + """ + if not self._available: + return None + + try: + brightness = self._brightness.get_brightness() + # get_brightness() returns a list if multiple displays + if isinstance(brightness, list): + return int(brightness[0]) + return int(brightness) + except Exception as e: + logger.error(f"Failed to get brightness: {e}") + return None + + def set_brightness(self, level: int) -> bool: + """Set display brightness (0-100). + + Args: + level: Brightness percentage 0-100. + + Returns: + True if successful, False otherwise. + """ + if not self._available: + return False + + level = max(0, min(100, level)) + + try: + self._brightness.set_brightness(level) + logger.info(f"Set brightness to {level}%") + return True + except Exception as e: + logger.error(f"Failed to set brightness: {e}") + return False + + def increase_brightness(self, amount: int = 5) -> Optional[int]: + """Increase brightness by amount. + + Args: + amount: Amount to increase (percentage points). + + Returns: + New brightness level, or None if failed. + """ + current = self.get_brightness() + if current is None: + return None + + new_level = min(100, current + amount) + if self.set_brightness(new_level): + return new_level + return None + + def decrease_brightness(self, amount: int = 5) -> Optional[int]: + """Decrease brightness by amount. + + Args: + amount: Amount to decrease (percentage points). + + Returns: + New brightness level, or None if failed. + """ + current = self.get_brightness() + if current is None: + return None + + new_level = max(0, current - amount) + if self.set_brightness(new_level): + return new_level + return None + + +class PowerController: + """Control power settings on Windows.""" + + @staticmethod + def sleep() -> bool: + """Put system to sleep. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + subprocess.run(["rundll32.exe", "powrprof.dll,SetSuspendState", "0", "1", "0"], check=False) + logger.info("Sent sleep command") + return True + except Exception as e: + logger.error(f"Failed to sleep: {e}") + return False + + @staticmethod + def shutdown(delay: int = 0) -> bool: + """Shutdown system. + + Args: + delay: Delay in seconds before shutdown. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + if delay > 0: + subprocess.run(["shutdown", "/s", "/t", str(delay)], check=False) + logger.info(f"Scheduled shutdown in {delay} seconds") + else: + subprocess.run(["shutdown", "/s", "/t", "0"], check=False) + logger.info("Sent shutdown command") + return True + except Exception as e: + logger.error(f"Failed to shutdown: {e}") + return False + + @staticmethod + def restart(delay: int = 0) -> bool: + """Restart system. + + Args: + delay: Delay in seconds before restart. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + if delay > 0: + subprocess.run(["shutdown", "/r", "/t", str(delay)], check=False) + logger.info(f"Scheduled restart in {delay} seconds") + else: + subprocess.run(["shutdown", "/r", "/t", "0"], check=False) + logger.info("Sent restart command") + return True + except Exception as e: + logger.error(f"Failed to restart: {e}") + return False + + @staticmethod + def cancel_shutdown() -> bool: + """Cancel pending shutdown/restart. + + Returns: + True if command sent, False on error. + """ + if platform.system() != "Windows": + logger.error("Power control only available on Windows") + return False + + try: + subprocess.run(["shutdown", "/a"], check=False) + logger.info("Cancelled pending shutdown/restart") + return True + except Exception as e: + logger.error(f"Failed to cancel shutdown: {e}") + return False + + +# ── Linux / Raspberry Pi backends ──────────────────────────────────────────── + +def _run(cmd: list[str], timeout: float = 5.0) -> Optional[str]: + """Run *cmd*; return stdout on success, None on any failure.""" + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.error(f"{cmd[0]} failed: {exc}") + return None + if proc.returncode != 0: + logger.error(f"{' '.join(cmd)} exited {proc.returncode}: {proc.stderr.strip()}") + return None + return proc.stdout + + +class LinuxVolumeController: + """Master volume via ``pactl`` (PipeWire/PulseAudio) with ``amixer`` (ALSA) fallback.""" + + def __init__(self, mixer_control: str = "Master"): + self._pactl = shutil.which("pactl") + self._amixer = shutil.which("amixer") + self._control = mixer_control + if not (self._pactl or self._amixer): + logger.warning("Neither pactl nor amixer found — volume control unavailable") + else: + logger.info(f"Linux volume control via {'pactl' if self._pactl else 'amixer'}") + + def _amixer_control(self) -> str: + # Pi HDMI/headphone cards expose "PCM" or "Headphone" instead of "Master". + out = _run([self._amixer, "scontrols"]) or "" + names = [line.split("'")[1] for line in out.splitlines() if "'" in line] + for cand in (self._control, "Master", "PCM", "Headphone", "Speaker", "Digital"): + if cand in names: + return cand + return names[0] if names else self._control + + def get_volume(self) -> Optional[float]: + if self._pactl: + out = _run([self._pactl, "get-sink-volume", "@DEFAULT_SINK@"]) + if out: + for tok in out.replace("/", " ").split(): + if tok.endswith("%") and tok[:-1].isdigit(): + return min(1.0, int(tok[:-1]) / 100.0) + if self._amixer: + out = _run([self._amixer, "get", self._amixer_control()]) + if out: + import re + m = re.search(r"\[(\d+)%\]", out) + if m: + return int(m.group(1)) / 100.0 + return None + + def set_volume(self, level: float) -> bool: + level = max(0.0, min(1.0, level)) + pct = f"{int(round(level * 100))}%" + if self._pactl and _run([self._pactl, "set-sink-volume", "@DEFAULT_SINK@", pct]) is not None: + logger.info(f"Set volume to {pct}") + return True + if self._amixer and _run([self._amixer, "set", self._amixer_control(), pct]) is not None: + logger.info(f"Set volume to {pct}") + return True + return False + + def is_muted(self) -> Optional[bool]: + if self._pactl: + out = _run([self._pactl, "get-sink-mute", "@DEFAULT_SINK@"]) + if out: + return "yes" in out.lower() + if self._amixer: + out = _run([self._amixer, "get", self._amixer_control()]) + if out: + return "[off]" in out + return None + + def set_mute(self, muted: bool) -> bool: + if self._pactl and _run([self._pactl, "set-sink-mute", "@DEFAULT_SINK@", "1" if muted else "0"]) is not None: + return True + if self._amixer and _run([self._amixer, "set", self._amixer_control(), "mute" if muted else "unmute"]) is not None: + return True + return False + + def increase_volume(self, amount: float = 0.05) -> Optional[float]: + cur = self.get_volume() + if cur is None: + return None + new = min(1.0, cur + amount) + return new if self.set_volume(new) else None + + def decrease_volume(self, amount: float = 0.05) -> Optional[float]: + cur = self.get_volume() + if cur is None: + return None + new = max(0.0, cur - amount) + return new if self.set_volume(new) else None + + +class LinuxBrightnessController: + """Backlight via ``/sys/class/backlight/*`` (official Pi touch display, DSI/eDP panels). + + Writing needs permission: add the user to the ``video`` group and install + the udev rule from RASPBERRY_PI.md, or run as root. + """ + + def __init__(self): + self._dev: Optional[Path] = None + for cand in sorted(Path("/sys/class/backlight").glob("*")): + if (cand / "brightness").exists() and (cand / "max_brightness").exists(): + self._dev = cand + break + if self._dev is None: + logger.warning("No /sys/class/backlight device — brightness control unavailable") + else: + logger.info(f"Linux brightness control via {self._dev}") + + def _max(self) -> int: + return max(1, int((self._dev / "max_brightness").read_text().strip())) # type: ignore[operator] + + def get_brightness(self) -> Optional[int]: + if self._dev is None: + return None + try: + cur = int((self._dev / "brightness").read_text().strip()) + return int(round(cur * 100 / self._max())) + except (OSError, ValueError) as exc: + logger.error(f"Failed to read brightness: {exc}") + return None + + def set_brightness(self, level: int) -> bool: + if self._dev is None: + return False + level = max(0, min(100, level)) + raw = int(round(level * self._max() / 100)) + try: + (self._dev / "brightness").write_text(str(raw)) + logger.info(f"Set brightness to {level}%") + return True + except OSError as exc: + logger.error(f"Failed to set brightness (permissions? see RASPBERRY_PI.md): {exc}") + return False + + def increase_brightness(self, amount: int = 5) -> Optional[int]: + cur = self.get_brightness() + if cur is None: + return None + new = min(100, cur + amount) + return new if self.set_brightness(new) else None + + def decrease_brightness(self, amount: int = 5) -> Optional[int]: + cur = self.get_brightness() + if cur is None: + return None + new = max(0, cur - amount) + return new if self.set_brightness(new) else None + + +class LinuxPowerController: + """Power via ``systemctl`` / ``shutdown`` (needs polkit rights or passwordless sudo — see RASPBERRY_PI.md).""" + + @staticmethod + def _priv(cmd: list[str]) -> list[str]: + import os + if os.geteuid() == 0 or not shutil.which("sudo"): + return cmd + return ["sudo", "-n", *cmd] + + def sleep(self) -> bool: + return _run(self._priv(["systemctl", "suspend"])) is not None + + def shutdown(self, delay: int = 0) -> bool: + when = f"+{max(1, (delay + 59) // 60)}" if delay > 0 else "now" + return _run(self._priv(["shutdown", "-h", when])) is not None + + def restart(self, delay: int = 0) -> bool: + when = f"+{max(1, (delay + 59) // 60)}" if delay > 0 else "now" + return _run(self._priv(["shutdown", "-r", when])) is not None + + def cancel_shutdown(self) -> bool: + return _run(self._priv(["shutdown", "-c"])) is not None + + +# Global controllers — one backend per platform. +if _IS_LINUX: + _volume = LinuxVolumeController() + _brightness = LinuxBrightnessController() + _power = LinuxPowerController() +else: + _volume = VolumeController() + _brightness = BrightnessController() + _power = PowerController() + + +def get_volume_controller() -> VolumeController: + """Get global volume controller.""" + return _volume + + +def get_brightness_controller() -> BrightnessController: + """Get global brightness controller.""" + return _brightness + + +def get_power_controller() -> PowerController: + """Get global power controller.""" + return _power diff --git a/mcp_servers.example.json b/mcp_servers.example.json new file mode 100644 index 0000000..50213f4 --- /dev/null +++ b/mcp_servers.example.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://example.com/mcp.schema.json", + "_comment": "VS Code-compatible MCP server config. Copy to mcp_servers.json and edit.", + "servers": { + "filesystem": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "c:\\local\\Code\\PRIVAT"], + "transport": "stdio", + "approval": "ask" + }, + "fetch": { + "command": "uvx", + "args": ["--with", "mcp==1.27.1", "mcp-server-fetch"], + "transport": "stdio", + "approval": "allow" + }, + "time": { + "command": "uvx", + "args": ["--with", "mcp==1.27.1", "mcp-server-time"], + "transport": "stdio", + "approval": "allow" + } + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c7bf46c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "javis" +version = "0.1.0" +description = "Local voice AI agent — wake-word driven, Ollama-backed, MCP-enabled, with voice cloning." +readme = "README.md" +requires-python = ">=3.10" +authors = [{ name = "Dani" }] +license = { text = "MIT" } +dependencies = [ + "pyyaml>=6.0", + "ruamel.yaml>=0.18", + "loguru>=0.7", + "httpx>=0.27", + "numpy>=1.24,<2.0", + "sounddevice>=0.4.6", + "soundfile>=0.12", + "scipy>=1.11", + "silero-vad>=5.1", + "openwakeword>=0.6.0", + "onnxruntime>=1.17", + "faster-whisper>=1.0", + "speechbrain>=1.0", + "piper-tts>=1.7.0,<2.0", + "transformers>=4.42,<4.50", + "mcp>=1.2", + "PySide6>=6.6", + "qasync>=0.27", + "pydantic>=2.5", + "fastapi>=0.110", + "uvicorn>=0.29", + "python-multipart>=0.0.9", +] + +[project.optional-dependencies] +cuda = ["torch>=2.2", "torchaudio>=2.2"] +dev = ["pytest>=7", "ruff>=0.4"] + +[project.scripts] +javis = "javis.__main__:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 110 +target-version = "py310" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2a0546b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,69 @@ +# Core +pyyaml>=6.0 +ruamel.yaml>=0.18 # comment-preserving config.yaml writes from the GUI +loguru>=0.7 +httpx>=0.27 +numpy>=1.24,<2.0 +pydantic>=2.5 + +# Audio I/O +sounddevice>=0.4.6 +soundfile>=0.12 +scipy>=1.11 + +# Wake word + VAD +silero-vad>=5.1 +openwakeword>=0.6.0 +onnxruntime>=1.17 + +# STT +# BatchedInferencePipeline (4× GPU speedup) requires >=1.1; Silero-VAD v6 in >=1.2.1 +faster-whisper>=1.1 + +# HuggingFace – hf_xet enables fast Xet-storage downloads (avoids HTTP fallback warning) +hf_xet>=1.0 + +# Speaker identification +speechbrain>=1.0 + +# TTS — Piper VITS (CPU ONNX inference, fast and low-overhead) +# piper-tts 2.x requires Python >=3.11; 1.7.0 has the same API surface +piper-tts>=1.7.0,<2.0 +# For voice training only (not needed for inference): +# pip install piper-train +# espeak-ng is required for training (German phonemisation): +# https://github.com/espeak-ng/espeak-ng/releases — install the Windows binary and add to PATH + +# MCP +mcp>=1.2 + +# Slave-device API server (javis-slave companion app) +fastapi>=0.110 +uvicorn>=0.29 +python-multipart>=0.0.9 + +# Optional: F5-TTS voice cloning backend (pip install f5-tts) +# Requires CC-BY-NC model weights downloaded from Hugging Face on first run. +# Switch backend via: tts.backend: f5 in config.yaml +# f5-tts>=1.1.20 + +# Optional: DeepFilterNet neural noise reduction (pip install deepfilternet) +# Automatically denoises mic input before STT when installed. +# deepfilternet>=0.5.6 + +# GUI +PySide6>=6.6 +qasync>=0.27 + +# CUDA (install separately based on driver version) +# CUDA 13.2 (current): pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu132 +# CUDA 12.4: pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu124 +# CUDA 12.1: pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu121 +# Or let the app auto-detect: python -m javis --install-deps +# +# NOTE: CTranslate2 (faster-whisper STT) requires cublas64_12.dll at runtime. +# Option A (pip — lightweight, recommended): +# pip install nvidia-cublas-cu12 nvidia-cudnn-cu12 nvidia-cuda-runtime-cu12 +# Option B: install the full CUDA 12 Toolkit from NVIDIA: +# https://developer.nvidia.com/cuda-12-0-1-download-archive +# Javis auto-discovers option-A DLLs via os.add_dll_directory() at startup. diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..4fccd95 --- /dev/null +++ b/run.bat @@ -0,0 +1,109 @@ +@echo off +REM Javis launcher (Windows) - hardware-aware bootstrap +setlocal +cd /d "%~dp0" + +set VENV=%~dp0.venv + +set FRESH_INSTALL=0 +REM A valid venv has activate.bat and pip.exe; python.exe alone means broken/partial venv. +if not exist "%VENV%\Scripts\activate.bat" goto :create_venv +if not exist "%VENV%\Scripts\pip.exe" goto :create_venv +goto :venv_ready + +:create_venv +if exist "%VENV%" ( + echo Removing incomplete virtual environment... + rmdir /s /q "%VENV%" +) +echo Creating virtual environment in %VENV%... +py -3 -m venv "%VENV%" 2>NUL +if errorlevel 1 ( + python -m venv "%VENV%" + if errorlevel 1 ( + echo [ERROR] Could not create venv. Is Python 3.10+ installed? + exit /b 1 + ) +) +set FRESH_INSTALL=1 + +:venv_ready + +call "%VENV%\Scripts\activate.bat" + +REM Self-heal: trigger full install if the venv exists but javis is not installed. +"%VENV%\Scripts\python.exe" -c "import javis" >NUL 2>&1 +if errorlevel 1 set FRESH_INSTALL=1 + +if not "%FRESH_INSTALL%"=="1" goto :ensure_mcp_servers + +echo Installing base dependencies... +"%VENV%\Scripts\python.exe" -m pip install --upgrade pip wheel setuptools +"%VENV%\Scripts\pip.exe" install -r requirements.txt +"%VENV%\Scripts\pip.exe" install -e . + +echo. +echo Detecting hardware and installing matching PyTorch wheel... +"%VENV%\Scripts\python.exe" -m javis --install-deps +if errorlevel 1 ( + echo. + echo [WARN] Bootstrap reported problems. You can retry with: + echo "%VENV%\Scripts\python.exe" -m javis --repair + echo. +) + +:ensure_espeak +REM Ensure piper-tts is installed (auto-heal if venv predates the Piper migration) +"%VENV%\Scripts\python.exe" -c "import piper" >NUL 2>&1 +if errorlevel 1 ( + echo Installing piper-tts... + "%VENV%\Scripts\pip.exe" install piper-tts +) + +REM Ensure espeak-ng is on PATH (needed for Piper training) +where espeak-ng >NUL 2>&1 +if not errorlevel 1 goto :ensure_mcp_servers +if exist "C:\Program Files\eSpeak NG\espeak-ng.exe" ( + set "PATH=C:\Program Files\eSpeak NG;%PATH%" + goto :ensure_mcp_servers +) +if "%FRESH_INSTALL%"=="1" ( + echo [WARN] espeak-ng not found on PATH. + echo Voice training requires espeak-ng. Download from: + echo https://github.com/espeak-ng/espeak-ng/releases +) + +:ensure_mcp_servers +REM Install each MCP sub-project into its own venv if not already present. +call :ensure_subvenv "homey-mcp" "homey_mcp" +call :ensure_subvenv "duckduckgo-mcp" "duckduckgo_mcp" +call :ensure_subvenv "emby-mcp" "emby_mcp" + +call :ensure_subvenv "javis-slave-mcp" "javis_slave_mcp" + +:run_app +if not exist "config.yaml" copy config.example.yaml config.yaml >NUL +if not exist "mcp_servers.json" copy mcp_servers.example.json mcp_servers.json >NUL + +"%VENV%\Scripts\python.exe" -m javis %* +endlocal +goto :EOF + +:ensure_subvenv +REM Usage: call :ensure_subvenv +REM Always uses 'python -m pip' to avoid broken pip.exe launchers after a directory move. +setlocal +set SDIR=%~dp0%~1 +set SVENV=%SDIR%\.venv +if not exist "%SVENV%\Scripts\python.exe" ( + echo Creating venv for %~1... + python -m venv "%SVENV%" + "%SVENV%\Scripts\python.exe" -m pip install --upgrade pip >NUL +) +"%SVENV%\Scripts\python.exe" -c "import %~2" >NUL 2>&1 +if errorlevel 1 ( + echo Installing %~1... + "%SVENV%\Scripts\python.exe" -m pip install -e "%SDIR%" +) +endlocal +goto :EOF diff --git a/spotify-mcp-server/.github/dependabot.yml b/spotify-mcp-server/.github/dependabot.yml new file mode 100644 index 0000000..bb39edf --- /dev/null +++ b/spotify-mcp-server/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + groups: + dependencies: + patterns: + - "*" \ No newline at end of file diff --git a/spotify-mcp-server/.github/workflows/ci.yml b/spotify-mcp-server/.github/workflows/ci.yml new file mode 100644 index 0000000..b272c53 --- /dev/null +++ b/spotify-mcp-server/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + lint-typecheck: + name: Lint and Typecheck + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Cache npm dependencies + uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm- + + - name: Install dependencies + run: npm install + + - name: Run Biome linting + run: npm run lint + + - name: Run TypeScript type checking + run: npm run typecheck \ No newline at end of file diff --git a/spotify-mcp-server/.gitignore b/spotify-mcp-server/.gitignore new file mode 100644 index 0000000..60fb537 --- /dev/null +++ b/spotify-mcp-server/.gitignore @@ -0,0 +1,8 @@ +node_modules +.vscode +.env +package-lock.json +spotify-config.json +build +dist +.DS_Store diff --git a/spotify-mcp-server/README.md b/spotify-mcp-server/README.md new file mode 100644 index 0000000..e9ceb83 --- /dev/null +++ b/spotify-mcp-server/README.md @@ -0,0 +1,411 @@ +
+ +

Spotify MCP Server

+
+ +A lightweight [Model Context Protocol (MCP)](https://modelcontextprotocol.io) server that enables AI assistants like Cursor & Claude to control Spotify playback and manage playlists. + +
+Contents + +- [Example Interactions](#example-interactions) +- [Tools](#tools) + - [Read Operations](#read-operations) + - [Album Operations](#album-operations) + - [Play / Create Operations](#play--create-operations) + - [Playlist Operations](#playlist-operations) +- [Setup](#setup) + - [Prerequisites](#prerequisites) + - [Installation](#installation) + - [Creating a Spotify Developer Application](#creating-a-spotify-developer-application) + - [Spotify API Configuration](#spotify-api-configuration) + - [Authentication Process](#authentication-process) +- [Integrating with Claude Desktop, Cursor, and VsCode (Cline)](#integrating-with-claude-desktop-and-cursor) +
+ +## Example Interactions + +- _"Play Elvis's first song"_ +- _"Create a Taylor Swift / Slipknot fusion playlist"_ +- _"Copy all the techno tracks from my workout playlist to my work playlist"_ +- _"Turn the volume down a bit"_ + +## Tools + +### Read Operations + +1. **searchSpotify** + + - **Description**: Search for tracks, albums, artists, or playlists on Spotify + - **Parameters**: + - `query` (string): The search term + - `type` (string): Type of item to search for (track, album, artist, playlist) + - `limit` (number, optional): Maximum number of results to return (10-50) + - **Returns**: List of matching items with their IDs, names, and additional details + - **Example**: `searchSpotify("bohemian rhapsody", "track", 20)` + +2. **getNowPlaying** + + - **Description**: Get information about the currently playing track on Spotify, including device and volume info + - **Parameters**: None + - **Returns**: Object containing track name, artist, album, playback progress, duration, playback state, device info, volume, and shuffle/repeat status + - **Example**: `getNowPlaying()` + +3. **getMyPlaylists** + + - **Description**: Get a list of the current user's playlists on Spotify + - **Parameters**: + - `limit` (number, optional): Maximum number of playlists to return (default: 20) + - `offset` (number, optional): Index of the first playlist to return (default: 0) + - **Returns**: Array of playlists with their IDs, names, track counts, and public status + - **Example**: `getMyPlaylists(10, 0)` + +4. **getPlaylistTracks** + + - **Description**: Get a list of tracks in a specific Spotify playlist + - **Parameters**: + - `playlistId` (string): The Spotify ID of the playlist + - `limit` (number, optional): Maximum number of tracks to return (default: 100) + - `offset` (number, optional): Index of the first track to return (default: 0) + - **Returns**: Array of tracks with their IDs, names, artists, album, duration, and added date + - **Example**: `getPlaylistTracks("37i9dQZEVXcJZyENOWUFo7")` + +5. **getRecentlyPlayed** + + - **Description**: Retrieves a list of recently played tracks from Spotify. + - **Parameters**: + - `limit` (number, optional): A number specifying the maximum number of tracks to return. + - **Returns**: If tracks are found it returns a formatted list of recently played tracks else a message stating: "You don't have any recently played tracks on Spotify". + - **Example**: `getRecentlyPlayed({ limit: 10 })` + +6. **getUsersSavedTracks** + + - **Description**: Get a list of tracks saved in the user's "Liked Songs" library + - **Parameters**: + - `limit` (number, optional): Maximum number of tracks to return (1-50, default: 50) + - `offset` (number, optional): Offset for pagination (0-based index, default: 0) + - **Returns**: Formatted list of saved tracks with track names, artists, duration, track IDs, and when they were added to Liked Songs. Shows pagination info (e.g., "1-20 of 150"). + - **Example**: `getUsersSavedTracks({ limit: 20, offset: 0 })` + +7. **getQueue** + + - **Description**: Get the currently playing track and upcoming items in the Spotify queue + - **Parameters**: + - `limit` (number, optional): Maximum number of upcoming items to show (1-50, default: 10) + - **Returns**: Currently playing track and list of upcoming tracks in the queue + - **Example**: `getQueue({ limit: 20 })` + +8. **getAvailableDevices** + + - **Description**: Get information about the user's available Spotify Connect devices + - **Parameters**: None + - **Returns**: List of available devices with name, type, active status, volume, and device ID + - **Example**: `getAvailableDevices()` + +9. **removeUsersSavedTracks** + + - **Description**: Remove one or more tracks from the user's "Liked Songs" library (max 40 per request) + - **Parameters**: + - `trackIds` (array): Array of Spotify track IDs to remove (max 40) + - **Returns**: Success confirmation message + - **Example**: `removeUsersSavedTracks({ trackIds: ["4iV5W9uYEdYUVa79Axb7Rh", "1301WleyT98MSxVHPZCA6M"] })` + + +### Play / Create Operations + +1. **playMusic** + + - **Description**: Start playing a track, album, artist, or playlist on Spotify + - **Parameters**: + - `uri` (string, optional): Spotify URI of the item to play (overrides type and id) + - `type` (string, optional): Type of item to play (track, album, artist, playlist) + - `id` (string, optional): Spotify ID of the item to play + - `deviceId` (string, optional): ID of the device to play on + - **Returns**: Success status + - **Example**: `playMusic({ uri: "spotify:track:6rqhFgbbKwnb9MLmUQDhG6" })` + - **Alternative**: `playMusic({ type: "track", id: "6rqhFgbbKwnb9MLmUQDhG6" })` + +2. **pausePlayback** + + - **Description**: Pause the currently playing track on Spotify + - **Parameters**: + - `deviceId` (string, optional): ID of the device to pause + - **Returns**: Success status + - **Example**: `pausePlayback()` + +3. **resumePlayback** + + - **Description**: Resume Spotify playback on the active device + - **Parameters**: + - `deviceId` (string, optional): ID of the device to resume playback on + - **Returns**: Success status + - **Example**: `resumePlayback()` + +4. **skipToNext** + + - **Description**: Skip to the next track in the current playback queue + - **Parameters**: + - `deviceId` (string, optional): ID of the device + - **Returns**: Success status + - **Example**: `skipToNext()` + +5. **skipToPrevious** + + - **Description**: Skip to the previous track in the current playback queue + - **Parameters**: + - `deviceId` (string, optional): ID of the device + - **Returns**: Success status + - **Example**: `skipToPrevious()` + +6. **createPlaylist** + + - **Description**: Create a new playlist on Spotify + - **Parameters**: + - `name` (string): Name for the new playlist + - `description` (string, optional): Description for the playlist + - `public` (boolean, optional): Whether the playlist should be public (default: false) + - **Returns**: Object with the new playlist's ID and URL + - **Example**: `createPlaylist({ name: "Workout Mix", description: "Songs to get pumped up", public: false })` + +7. **addTracksToPlaylist** + + - **Description**: Add tracks to an existing Spotify playlist + - **Parameters**: + - `playlistId` (string): ID of the playlist + - `trackUris` (array): Array of track URIs or IDs to add + - `position` (number, optional): Position to insert tracks + - **Returns**: Success status and snapshot ID + - **Example**: `addTracksToPlaylist({ playlistId: "3cEYpjA9oz9GiPac4AsH4n", trackUris: ["spotify:track:4iV5W9uYEdYUVa79Axb7Rh"] })` + +8. **addToQueue** + + - **Description**: Adds a track, album, artist or playlist to the current playback queue + - **Parameters**: + - `uri` (string, optional): Spotify URI of the item to add to queue (overrides type and id) + - `type` (string, optional): Type of item to queue (track, album, artist, playlist) + - `id` (string, optional): Spotify ID of the item to queue + - `deviceId` (string, optional): ID of the device to queue on + - **Returns**: Success status + - **Example**: `addToQueue({ uri: "spotify:track:6rqhFgbbKwnb9MLmUQDhG6" })` + - **Alternative**: `addToQueue({ type: "track", id: "6rqhFgbbKwnb9MLmUQDhG6" })` + +9. **setVolume** + + - **Description**: Set the playback volume to a specific percentage (requires Spotify Premium) + - **Parameters**: + - `volumePercent` (number): The volume to set (0-100) + - `deviceId` (string, optional): ID of the device to set volume on + - **Returns**: Success status with the new volume level + - **Example**: `setVolume({ volumePercent: 50 })` + +10. **adjustVolume** + + - **Description**: Adjust the playback volume up or down by a relative amount (requires Spotify Premium) + - **Parameters**: + - `adjustment` (number): The amount to adjust volume by (-100 to 100). Positive values increase volume, negative values decrease it. + - `deviceId` (string, optional): ID of the device to adjust volume on + - **Returns**: Success status showing the volume change (e.g., "Volume increased from 50% to 60%") + - **Example**: `adjustVolume({ adjustment: 10 })` (increase by 10%) + - **Example**: `adjustVolume({ adjustment: -20 })` (decrease by 20%) + + +### Album Operations + +1. **getAlbums** + + - **Description**: Get detailed information about one or more albums by their Spotify IDs + - **Parameters**: + - `albumIds` (string|array): A single album ID or array of album IDs (max 20) + - **Returns**: Album details including name, artists, release date, type, total tracks, and ID. For single album returns detailed view, for multiple albums returns summary list. + - **Example**: `getAlbums("4aawyAB9vmqN3uQ7FjRGTy")` or `getAlbums(["4aawyAB9vmqN3uQ7FjRGTy", "1DFixLWuPkv3KT3TnV35m3"])` + +2. **getAlbumTracks** + + - **Description**: Get tracks from a specific album with pagination support + - **Parameters**: + - `albumId` (string): The Spotify ID of the album + - `limit` (number, optional): Maximum number of tracks to return (1-50) + - `offset` (number, optional): Offset for pagination (0-based index) + - **Returns**: List of tracks from the album with track names, artists, duration, and IDs. Shows pagination info. + - **Example**: `getAlbumTracks("4aawyAB9vmqN3uQ7FjRGTy", 10, 0)` + +3. **saveOrRemoveAlbumForUser** + + - **Description**: Save or remove albums from the user's "Your Music" library + - **Parameters**: + - `albumIds` (array): Array of Spotify album IDs (max 20) + - `action` (string): Action to perform: "save" or "remove" + - **Returns**: Success status with confirmation message + - **Example**: `saveOrRemoveAlbumForUser(["4aawyAB9vmqN3uQ7FjRGTy"], "save")` + +4. **checkUsersSavedAlbums** + + - **Description**: Check if albums are saved in the user's "Your Music" library + - **Parameters**: + - `albumIds` (array): Array of Spotify album IDs to check (max 20) + - **Returns**: Status of each album (saved or not saved) + - **Example**: `checkUsersSavedAlbums(["4aawyAB9vmqN3uQ7FjRGTy", "1DFixLWuPkv3KT3TnV35m3"])` + +### Playlist Operations + +1. **getPlaylist** + + - **Description**: Get details of a specific Spotify playlist including tracks count, description and owner + - **Parameters**: + - `playlistId` (string): The Spotify ID of the playlist + - **Returns**: Playlist name, owner, track count, visibility, description, ID, and URL + - **Example**: `getPlaylist({ playlistId: "37i9dQZEVXcJZyENOWUFo7" })` + +2. **updatePlaylist** + + - **Description**: Update the details of a Spotify playlist (name, description, public/private, collaborative) + - **Parameters**: + - `playlistId` (string): The Spotify ID of the playlist + - `name` (string, optional): New name for the playlist + - `description` (string, optional): New description for the playlist + - `public` (boolean, optional): Whether the playlist should be public + - `collaborative` (boolean, optional): Whether the playlist should be collaborative (requires public to be false) + - **Returns**: Success confirmation with list of updated fields + - **Example**: `updatePlaylist({ playlistId: "3cEYpjA9oz9GiPac4AsH4n", name: "New Name", public: true })` + +3. **removeTracksFromPlaylist** + + - **Description**: Remove one or more tracks from a Spotify playlist (max 100 tracks per request) + - **Parameters**: + - `playlistId` (string): The Spotify ID of the playlist + - `trackIds` (array): Array of Spotify track IDs to remove (max 100) + - `snapshotId` (string, optional): The playlist snapshot ID to target a specific version + - **Returns**: Success confirmation with the number of tracks removed + - **Example**: `removeTracksFromPlaylist({ playlistId: "3cEYpjA9oz9GiPac4AsH4n", trackIds: ["4iV5W9uYEdYUVa79Axb7Rh"] })` + +4. **reorderPlaylistItems** + + - **Description**: Reorder a range of tracks within a Spotify playlist by moving them to a new position + - **Parameters**: + - `playlistId` (string): The Spotify ID of the playlist + - `rangeStart` (number): The position of the first item to move (0-based index) + - `insertBefore` (number): The position where the items should be inserted (0-based index) + - `rangeLength` (number, optional): Number of consecutive items to move (defaults to 1) + - `snapshotId` (string, optional): The playlist snapshot ID to target a specific version + - **Returns**: Success confirmation with the move details + - **Example**: `reorderPlaylistItems({ playlistId: "3cEYpjA9oz9GiPac4AsH4n", rangeStart: 2, insertBefore: 0 })` + +## Setup + +### Prerequisites + +- Node.js v16+ +- A Spotify Premium account +- A registered Spotify Developer application + +### Installation + +```bash +git clone https://github.com/marcelmarais/spotify-mcp-server.git +cd spotify-mcp-server +npm install +npm run build +``` + +### Creating a Spotify Developer Application + +1. Go to the [Spotify Developer Dashboard](https://developer.spotify.com/dashboard/) +2. Log in with your Spotify account +3. Click the "Create an App" button +4. Fill in the app name and description +5. Accept the Terms of Service and click "Create" +6. In your new app's dashboard, you'll see your **Client ID** +7. Click "Show Client Secret" to reveal your **Client Secret** +8. Click "Edit Settings" and add a Redirect URI (e.g., `http://127.0.0.1:8888/callback`) +9. Save your changes + +### Spotify API Configuration + +Create a `spotify-config.json` file in the project root (you can copy and modify the provided example): + +```bash +# Copy the example config file +cp spotify-config.example.json spotify-config.json +``` + +Then edit the file with your credentials: + +```json +{ + "clientId": "your-client-id", + "clientSecret": "your-client-secret", + "redirectUri": "http://127.0.0.1:8888/callback" +} +``` + +### Authentication Process + +The Spotify API uses OAuth 2.0 for authentication. Follow these steps to authenticate your application: + +1. Run the authentication script: + +```bash +npm run auth +``` + +2. The script will generate an authorization URL. Open this URL in your web browser. + +3. You'll be prompted to log in to Spotify and authorize your application. + +4. After authorization, Spotify will redirect you to your specified redirect URI with a code parameter in the URL. + +5. The authentication script will automatically exchange this code for access and refresh tokens. + +6. These tokens will be saved to your `spotify-config.json` file, which will now look something like: + +```json +{ + "clientId": "your-client-id", + "clientSecret": "your-client-secret", + "redirectUri": "http://localhost:8888/callback", + "accessToken": "BQAi9Pn...kKQ", + "refreshToken": "AQDQcj...7w", + "expiresAt": 1677889354671 +} +``` + +**Note**: The `expiresAt` field is a Unix timestamp (in milliseconds) indicating when the access token expires. + +7. **Automatic Token Refresh**: The server will automatically refresh the access token when it expires (typically after 1 hour). The refresh happens transparently using the `refreshToken`, so you don't need to re-authenticate manually. If the refresh fails, you'll need to run `npm run auth` again to re-authenticate. + +## Integrating with Claude Desktop, Cursor, and VsCode [Via Cline model extension](https://marketplace.visualstudio.com/items/?itemName=saoudrizwan.claude-dev) + +To use your MCP server with Claude Desktop, add it to your Claude configuration: + +```json +{ + "mcpServers": { + "spotify": { + "command": "node", + "args": ["spotify-mcp-server/build/index.js"] + } + } +} +``` + +For Cursor, go to the MCP tab in `Cursor Settings` (command + shift + J). Add a server with this command: + +```bash +node path/to/spotify-mcp-server/build/index.js +``` + +To set up your MCP correctly with Cline ensure you have the following file configuration set `cline_mcp_settings.json`: + +```json +{ + "mcpServers": { + "spotify": { + "command": "node", + "args": ["~/../spotify-mcp-server/build/index.js"], + "autoApprove": ["getListeningHistory", "getNowPlaying"] + } + } +} +``` + +You can add additional tools to the auto approval array to run the tools without intervention. diff --git a/spotify-mcp-server/biome.jsonc b/spotify-mcp-server/biome.jsonc new file mode 100644 index 0000000..0c8379d --- /dev/null +++ b/spotify-mcp-server/biome.jsonc @@ -0,0 +1,149 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", + "files": { + "ignoreUnknown": false, + "ignore": [ + "**/pnpm-lock.yaml", + "lib/db/migrations", + "lib/editor/react-renderer.tsx", + "node_modules", + ".next", + "public", + ".vercel" + ] + }, + "vcs": { + "enabled": true, + "clientKind": "git", + "defaultBranch": "main", + "useIgnoreFile": true + }, + "formatter": { + "enabled": true, + "formatWithErrors": false, + "indentStyle": "space", + "indentWidth": 2, + "lineEnding": "lf", + "lineWidth": 80, + "attributePosition": "auto" + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "a11y": { + "useHtmlLang": "warn", // Not in recommended ruleset, turning on manually + "noHeaderScope": "warn", // Not in recommended ruleset, turning on manually + "useValidAriaRole": { + "level": "warn", + "options": { + "ignoreNonDom": false, + "allowInvalidRoles": ["none", "text"] + } + }, + "useSemanticElements": "off", // Rule is buggy, revisit later + "noSvgWithoutTitle": "off", // We do not intend to adhere to this rule + "useMediaCaption": "off", // We would need a cultural change to turn this on + "noAutofocus": "off", // We're highly intentional about when we use autofocus + "noBlankTarget": "off", // Covered by Conformance + "useFocusableInteractive": "off", // Disable focusable interactive element requirement + "useAriaPropsForRole": "off", // Disable required ARIA attributes check + "useKeyWithClickEvents": "off" // Disable keyboard event requirement with click events + }, + "complexity": { + "noUselessStringConcat": "warn", + "noForEach": "off", + "noUselessSwitchCase": "off", + "noUselessThisAlias": "off", + "noUselessTernary": "error", + "useFlatMap": "error", + "useOptionalChain": "error", + "useSimplifiedLogicExpression": "error" + }, + "correctness": { + "noUnusedImports": "error", + "useArrayLiterals": "warn", // Not in recommended ruleset, turning on manually + "noNewSymbol": "warn", // Not in recommended ruleset, turning on manually + "useJsxKeyInIterable": "off", // Rule is buggy, revisit later + "useExhaustiveDependencies": "off", // Community feedback on this rule has been poor, we will continue with ESLint + "noUnnecessaryContinue": "off", // Turned off due to developer preferences + "noUnusedVariables": "error" + }, + "security": { + "noDangerouslySetInnerHtml": "error" + }, + "style": { + "useFragmentSyntax": "warn", + "noYodaExpression": "warn", + "useDefaultParameterLast": "warn", + "useExponentiationOperator": "off", + "noUnusedTemplateLiteral": "off", + "noUselessElse": "error", + "useImportType": "error", + "useSelfClosingElements": "error", + "useFilenamingConvention": { + "level": "error", + "options": { + "filenameCases": ["kebab-case", "export"], + "requireAscii": true + } + } + }, + "suspicious": { + "noExplicitAny": "off" + }, + "nursery": { + "noStaticElementInteractions": "warn", + "noHeadImportInDocument": "warn", + "noDocumentImportInPage": "warn", + "noDuplicateElseIf": "warn", + "noIrregularWhitespace": "warn", + "useValidAutocomplete": "warn", + "useSortedClasses": "error" + } + } + }, + "javascript": { + "jsxRuntime": "reactClassic", + "formatter": { + "jsxQuoteStyle": "double", + "quoteProperties": "asNeeded", + "trailingCommas": "all", + "semicolons": "always", + "arrowParentheses": "always", + "bracketSpacing": true, + "bracketSameLine": false, + "quoteStyle": "single", + "attributePosition": "auto" + } + }, + "json": { + "formatter": { + "enabled": true, + "trailingCommas": "none" + }, + "parser": { + "allowComments": true, + "allowTrailingCommas": false + } + }, + "css": { + "formatter": { "enabled": false }, + "linter": { "enabled": false } + }, + "organizeImports": { "enabled": true }, + "overrides": [ + // Playwright requires an object destructure, even if empty + // https://github.com/microsoft/playwright/issues/30007 + { + "include": ["playwright/**"], + "linter": { + "rules": { + "correctness": { + "noEmptyPattern": "off" + } + } + } + } + ] +} diff --git a/spotify-mcp-server/package.json b/spotify-mcp-server/package.json new file mode 100644 index 0000000..03932d4 --- /dev/null +++ b/spotify-mcp-server/package.json @@ -0,0 +1,31 @@ +{ + "type": "module", + "name": "spotify-mcp-server", + "version": "1.0.0", + "main": "index.js", + "bin": { + "spotify-mcp": "./build/index.js" + }, + "scripts": { + "build": "tsc && node -e \"require('fs').chmodSync('build/index.js', '755')\"", + "auth": "tsc && node build/auth.js", + "lint": "biome check --diagnostic-level=warn --error-on-warnings", + "lint:fix": "biome check --write --unsafe --organize-imports-enabled=true", + "typecheck": "tsc --noEmit" + }, + "files": ["build"], + "keywords": [], + "author": "Marcel Marais", + "description": "A Model Context Protocol server for Spotify integration", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.10.1", + "@spotify/web-api-ts-sdk": "^1.2.0", + "open": "^10.1.0", + "zod": "^3.24.2" + }, + "devDependencies": { + "@biomejs/biome": "^1.9.4", + "@types/node": "^22.13.8", + "typescript": "^5.8.2" + } +} diff --git a/spotify-mcp-server/spotify-config.example.json b/spotify-mcp-server/spotify-config.example.json new file mode 100644 index 0000000..bddfc50 --- /dev/null +++ b/spotify-mcp-server/spotify-config.example.json @@ -0,0 +1,8 @@ +{ + "clientId": "your-client-id", + "clientSecret": "your-client-secret", + "redirectUri": "http://127.0.0.1:8888/callback", + "accessToken": "run-npm auth to get this", + "refreshToken": "run-npm auth to get this", + "expiresAt": "automatically set during authentication" +} diff --git a/spotify-mcp-server/src/albums.ts b/spotify-mcp-server/src/albums.ts new file mode 100644 index 0000000..5a5fc78 --- /dev/null +++ b/spotify-mcp-server/src/albums.ts @@ -0,0 +1,304 @@ +import type { MaxInt } from '@spotify/web-api-ts-sdk'; +import { z } from 'zod'; +import type { SpotifyHandlerExtra, tool } from './types.js'; +import { formatDuration, handleSpotifyRequest } from './utils.js'; + +const getAlbums: tool<{ + albumIds: z.ZodUnion<[z.ZodString, z.ZodArray]>; +}> = { + name: 'getAlbums', + description: + 'Get detailed information about one or more albums by their Spotify IDs', + schema: { + albumIds: z + .union([z.string(), z.array(z.string()).max(20)]) + .describe('A single album ID or array of album IDs (max 20)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { albumIds } = args; + const ids = Array.isArray(albumIds) ? albumIds : [albumIds]; + + if (ids.length === 0) { + return { + content: [ + { + type: 'text', + text: 'Error: No album IDs provided', + }, + ], + }; + } + + try { + const albums = await handleSpotifyRequest(async (spotifyApi) => { + return ids.length === 1 + ? [await spotifyApi.albums.get(ids[0])] + : await spotifyApi.albums.get(ids); + }); + + if (albums.length === 0) { + return { + content: [ + { + type: 'text', + text: 'No albums found for the provided IDs', + }, + ], + }; + } + + if (albums.length === 1) { + const album = albums[0]; + const artists = album.artists.map((a) => a.name).join(', '); + const releaseDate = album.release_date; + const totalTracks = album.total_tracks; + const albumType = album.album_type; + + return { + content: [ + { + type: 'text', + text: `# Album Details\n\n**Name**: "${album.name}"\n**Artists**: ${artists}\n**Release Date**: ${releaseDate}\n**Type**: ${albumType}\n**Total Tracks**: ${totalTracks}\n**ID**: ${album.id}`, + }, + ], + }; + } + + const formattedAlbums = albums + .map((album, i) => { + if (!album) return `${i + 1}. [Album not found]`; + + const artists = album.artists.map((a) => a.name).join(', '); + return `${i + 1}. "${album.name}" by ${artists} (${album.release_date}) - ${album.total_tracks} tracks - ID: ${album.id}`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Multiple Albums\n\n${formattedAlbums}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting albums: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getAlbumTracks: tool<{ + albumId: z.ZodString; + limit: z.ZodOptional; + offset: z.ZodOptional; +}> = { + name: 'getAlbumTracks', + description: 'Get tracks from a specific album with pagination support', + schema: { + albumId: z.string().describe('The Spotify ID of the album'), + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of tracks to return (1-50)'), + offset: z + .number() + .min(0) + .optional() + .describe('Offset for pagination (0-based index)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { albumId, limit = 20, offset = 0 } = args; + + try { + const tracks = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.albums.tracks( + albumId, + undefined, + limit as MaxInt<50>, + offset, + ); + }); + + if (tracks.items.length === 0) { + return { + content: [ + { + type: 'text', + text: 'No tracks found in this album', + }, + ], + }; + } + + const formattedTracks = tracks.items + .map((track, i) => { + if (!track) return `${i + 1}. [Track not found]`; + + const artists = track.artists.map((a) => a.name).join(', '); + const duration = formatDuration(track.duration_ms); + return `${offset + i + 1}. "${track.name}" by ${artists} (${duration}) - ID: ${track.id}`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Album Tracks (${offset + 1}-${offset + tracks.items.length} of ${tracks.total})\n\n${formattedTracks}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting album tracks: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const saveOrRemoveAlbumForUser: tool<{ + albumIds: z.ZodArray; + action: z.ZodEnum<['save', 'remove']>; +}> = { + name: 'saveOrRemoveAlbumForUser', + description: 'Save or remove albums from the user\'s "Your Music" library', + schema: { + albumIds: z + .array(z.string()) + .max(20) + .describe('Array of Spotify album IDs (max 20)'), + action: z + .enum(['save', 'remove']) + .describe('Action to perform: save or remove albums'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { albumIds, action } = args; + + if (albumIds.length === 0) { + return { + content: [ + { + type: 'text', + text: 'Error: No album IDs provided', + }, + ], + }; + } + + try { + await handleSpotifyRequest(async (spotifyApi) => { + return action === 'save' + ? await spotifyApi.currentUser.albums.saveAlbums(albumIds) + : await spotifyApi.currentUser.albums.removeSavedAlbums(albumIds); + }); + + const actionPastTense = action === 'save' ? 'saved' : 'removed'; + const preposition = action === 'save' ? 'to' : 'from'; + + return { + content: [ + { + type: 'text', + text: `Successfully ${actionPastTense} ${albumIds.length} album${albumIds.length === 1 ? '' : 's'} ${preposition} your library`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error ${action === 'save' ? 'saving' : 'removing'} albums: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const checkUsersSavedAlbums: tool<{ + albumIds: z.ZodArray; +}> = { + name: 'checkUsersSavedAlbums', + description: 'Check if albums are saved in the user\'s "Your Music" library', + schema: { + albumIds: z + .array(z.string()) + .max(20) + .describe('Array of Spotify album IDs to check (max 20)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { albumIds } = args; + + if (albumIds.length === 0) { + return { + content: [ + { + type: 'text', + text: 'Error: No album IDs provided', + }, + ], + }; + } + + try { + const savedStatus = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.currentUser.albums.hasSavedAlbums(albumIds); + }); + + const formattedResults = albumIds + .map((albumId, i) => { + const isSaved = savedStatus[i]; + return `${i + 1}. ${albumId}: ${isSaved ? 'Saved' : 'Not saved'}`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Album Save Status\n\n${formattedResults}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error checking saved albums: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +export const albumTools = [ + getAlbums, + getAlbumTracks, + saveOrRemoveAlbumForUser, + checkUsersSavedAlbums, +]; diff --git a/spotify-mcp-server/src/auth.ts b/spotify-mcp-server/src/auth.ts new file mode 100644 index 0000000..58b1f1c --- /dev/null +++ b/spotify-mcp-server/src/auth.ts @@ -0,0 +1,14 @@ +#!/usr/bin/env node + +import { authorizeSpotify } from './utils.js'; + +console.log('Starting Spotify authentication flow...'); +authorizeSpotify() + .then(() => { + console.log('Authentication completed successfully!'); + process.exit(0); + }) + .catch((error) => { + console.error('Authentication failed:', error); + process.exit(1); + }); diff --git a/spotify-mcp-server/src/index.ts b/spotify-mcp-server/src/index.ts new file mode 100644 index 0000000..668f719 --- /dev/null +++ b/spotify-mcp-server/src/index.ts @@ -0,0 +1,27 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { albumTools } from './albums.js'; +import { playTools } from './play.js'; +import { playlistTools } from './playlist.js'; +import { readTools } from './read.js'; + +const server = new McpServer({ + name: 'spotify-controller', + version: '1.0.0', +}); + +[...readTools, ...playTools, ...albumTools, ...playlistTools].forEach( + (tool) => { + server.tool(tool.name, tool.description, tool.schema, tool.handler); + }, +); + +async function main() { + const transport = new StdioServerTransport(); + await server.connect(transport); +} + +main().catch((error) => { + console.error('Fatal error in main():', error); + process.exit(1); +}); diff --git a/spotify-mcp-server/src/play.ts b/spotify-mcp-server/src/play.ts new file mode 100644 index 0000000..d9dac75 --- /dev/null +++ b/spotify-mcp-server/src/play.ts @@ -0,0 +1,586 @@ +import { z } from 'zod'; +import type { SpotifyHandlerExtra, tool } from './types.js'; +import { handleSpotifyRequest, spotifyFetch } from './utils.js'; + +const playMusic: tool<{ + uri: z.ZodOptional; + type: z.ZodOptional>; + id: z.ZodOptional; + deviceId: z.ZodOptional; +}> = { + name: 'playMusic', + description: 'Start playing a Spotify track, album, artist, or playlist', + schema: { + uri: z + .string() + .optional() + .describe('The Spotify URI to play (overrides type and id)'), + type: z + .enum(['track', 'album', 'artist', 'playlist']) + .optional() + .describe('The type of item to play'), + id: z.string().optional().describe('The Spotify ID of the item to play'), + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to play on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { uri, type, id, deviceId } = args; + + if (!(uri || (type && id))) { + return { + content: [ + { + type: 'text', + text: 'Error: Must provide either a URI or both a type and ID', + }, + ], + }; + } + + let spotifyUri = uri; + if (!spotifyUri && type && id) { + spotifyUri = `spotify:${type}:${id}`; + } + + try { + await handleSpotifyRequest(async (spotifyApi) => { + const device = deviceId || ''; + + if (!spotifyUri) { + await spotifyApi.player.startResumePlayback(device); + return; + } + + if (type === 'track') { + await spotifyApi.player.startResumePlayback(device, undefined, [ + spotifyUri, + ]); + } else { + await spotifyApi.player.startResumePlayback(device, spotifyUri); + } + }); + + return { + content: [ + { + type: 'text', + text: `Started playing ${type || 'music'} ${id ? `(ID: ${id})` : ''}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error starting playback: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const pausePlayback: tool<{ + deviceId: z.ZodOptional; +}> = { + name: 'pausePlayback', + description: 'Pause Spotify playback on the active device', + schema: { + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to pause playback on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { deviceId } = args; + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.pausePlayback(deviceId || ''); + }); + + return { + content: [{ type: 'text', text: 'Playback paused' }], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error pausing playback: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const skipToNext: tool<{ + deviceId: z.ZodOptional; +}> = { + name: 'skipToNext', + description: 'Skip to the next track in the current Spotify playback queue', + schema: { + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to skip on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { deviceId } = args; + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.skipToNext(deviceId || ''); + }); + + return { + content: [{ type: 'text', text: 'Skipped to next track' }], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error skipping to next track: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const skipToPrevious: tool<{ + deviceId: z.ZodOptional; +}> = { + name: 'skipToPrevious', + description: + 'Skip to the previous track in the current Spotify playback queue', + schema: { + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to skip on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { deviceId } = args; + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.skipToPrevious(deviceId || ''); + }); + + return { + content: [{ type: 'text', text: 'Skipped to previous track' }], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error skipping to previous track: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const createPlaylist: tool<{ + name: z.ZodString; + description: z.ZodOptional; + public: z.ZodOptional; +}> = { + name: 'createPlaylist', + description: 'Create a new playlist on Spotify', + schema: { + name: z.string().describe('The name of the playlist'), + description: z + .string() + .optional() + .describe('The description of the playlist'), + public: z + .boolean() + .optional() + .describe('Whether the playlist should be public'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { name, description, public: isPublic = false } = args; + + try { + // Get user ID via raw fetch (SDK playlists.createPlaylist hit a 403 after + // the March 2026 Spotify API changes — use spotifyFetch to avoid the SDK + // OAuth signing layer and hit the endpoint directly with Bearer token). + const me = await spotifyFetch<{ id: string }>('me'); + const playlist = await spotifyFetch<{ + id: string; + external_urls: { spotify: string }; + }>(`users/${me.id}/playlists`, { + method: 'POST', + body: { + name, + ...(description !== undefined ? { description } : {}), + public: isPublic, + }, + }); + + return { + content: [ + { + type: 'text', + text: `Successfully created playlist "${name}"\nPlaylist ID: ${playlist.id}\nPlaylist URL: ${playlist.external_urls.spotify}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error creating playlist: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const addTracksToPlaylist: tool<{ + playlistId: z.ZodString; + trackIds: z.ZodArray; + position: z.ZodOptional; +}> = { + name: 'addTracksToPlaylist', + description: 'Add tracks to a Spotify playlist', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + trackIds: z.array(z.string()).describe('Array of Spotify track IDs to add'), + position: z + .number() + .nonnegative() + .optional() + .describe('Position to insert the tracks (0-based index)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { playlistId, trackIds, position } = args; + + if (trackIds.length === 0) { + return { + content: [ + { + type: 'text', + text: 'Error: No track IDs provided', + }, + ], + }; + } + + try { + const trackUris = trackIds.map((id) => `spotify:track:${id}`); + + // Hit /items directly: see spotifyFetch JSDoc for context. + await spotifyFetch(`playlists/${playlistId}/items`, { + method: 'POST', + body: { + uris: trackUris, + ...(position !== undefined ? { position } : {}), + }, + }); + + return { + content: [ + { + type: 'text', + text: `Successfully added ${trackIds.length} track${ + trackIds.length === 1 ? '' : 's' + } to playlist (ID: ${playlistId})`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error adding tracks to playlist: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const resumePlayback: tool<{ + deviceId: z.ZodOptional; +}> = { + name: 'resumePlayback', + description: 'Resume Spotify playback on the active device', + schema: { + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to resume playback on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { deviceId } = args; + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.startResumePlayback(deviceId || ''); + }); + + return { + content: [{ type: 'text', text: 'Playback resumed' }], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error resuming playback: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const addToQueue: tool<{ + uri: z.ZodOptional; + type: z.ZodOptional>; + id: z.ZodOptional; + deviceId: z.ZodOptional; +}> = { + name: 'addToQueue', + description: 'Adds a track, album, artist or playlist to the playback queue', + schema: { + uri: z + .string() + .optional() + .describe('The Spotify URI to play (overrides type and id)'), + type: z + .enum(['track', 'album', 'artist', 'playlist']) + .optional() + .describe('The type of item to play'), + id: z.string().optional().describe('The Spotify ID of the item to play'), + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to add the track to'), + }, + handler: async (args) => { + const { uri, type, id, deviceId } = args; + + let spotifyUri = uri; + if (!spotifyUri && type && id) { + spotifyUri = `spotify:${type}:${id}`; + } + + if (!spotifyUri) { + return { + content: [ + { + type: 'text', + text: 'Error: Must provide either a URI or both a type and ID', + }, + ], + }; + } + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.addItemToPlaybackQueue( + spotifyUri, + deviceId || '', + ); + }); + + return { + content: [ + { + type: 'text', + text: `Added item ${spotifyUri} to queue`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error adding to queue: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const setVolume: tool<{ + volumePercent: z.ZodNumber; + deviceId: z.ZodOptional; +}> = { + name: 'setVolume', + description: + 'Set the playback volume to a specific percentage (0-100). Requires Spotify Premium.', + schema: { + volumePercent: z + .number() + .min(0) + .max(100) + .describe('The volume to set (0-100)'), + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to set volume on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { volumePercent, deviceId } = args; + + try { + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.setPlaybackVolume( + Math.round(volumePercent), + deviceId || '', + ); + }); + + return { + content: [ + { + type: 'text', + text: `Volume set to ${Math.round(volumePercent)}%`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error setting volume: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const adjustVolume: tool<{ + adjustment: z.ZodNumber; + deviceId: z.ZodOptional; +}> = { + name: 'adjustVolume', + description: + 'Adjust the playback volume up or down by a relative amount. Use positive values to increase, negative to decrease. Requires Spotify Premium.', + schema: { + adjustment: z + .number() + .min(-100) + .max(100) + .describe( + 'The amount to adjust volume by (-100 to 100). Positive increases, negative decreases.', + ), + deviceId: z + .string() + .optional() + .describe('The Spotify device ID to adjust volume on'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { adjustment, deviceId } = args; + + try { + // First get the current playback state to find current volume + const playback = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.player.getPlaybackState(); + }); + + if (!playback?.device) { + return { + content: [ + { + type: 'text', + text: 'No active device found. Make sure Spotify is open and playing on a device.', + }, + ], + }; + } + + const currentVolume = playback.device.volume_percent; + if (currentVolume === null || currentVolume === undefined) { + return { + content: [ + { + type: 'text', + text: 'Unable to get current volume from device.', + }, + ], + }; + } + + const newVolume = Math.min(100, Math.max(0, currentVolume + adjustment)); + + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.player.setPlaybackVolume( + Math.round(newVolume), + deviceId || '', + ); + }); + + const direction = adjustment > 0 ? 'increased' : 'decreased'; + return { + content: [ + { + type: 'text', + text: `Volume ${direction} from ${currentVolume}% to ${Math.round(newVolume)}%`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error adjusting volume: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +export const playTools = [ + playMusic, + pausePlayback, + skipToNext, + skipToPrevious, + createPlaylist, + addTracksToPlaylist, + resumePlayback, + addToQueue, + setVolume, + adjustVolume, +]; diff --git a/spotify-mcp-server/src/playlist.ts b/spotify-mcp-server/src/playlist.ts new file mode 100644 index 0000000..e6cf11b --- /dev/null +++ b/spotify-mcp-server/src/playlist.ts @@ -0,0 +1,294 @@ +import { z } from 'zod'; +import type { SpotifyHandlerExtra, tool } from './types.js'; +import { handleSpotifyRequest, spotifyFetch } from './utils.js'; + +const getPlaylist: tool<{ + playlistId: z.ZodString; +}> = { + name: 'getPlaylist', + description: + 'Get details of a specific Spotify playlist including tracks count, description and owner', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { playlistId } = args; + + try { + const playlist = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.playlists.getPlaylist(playlistId); + }); + + const owner = + playlist.owner?.display_name ?? playlist.owner?.id ?? 'Unknown'; + const tracksTotal = playlist.tracks?.total ?? 0; + const isPublic = playlist.public ? 'Public' : 'Private'; + const isCollaborative = playlist.collaborative ? ' | Collaborative' : ''; + const description = playlist.description + ? `\n**Description**: ${playlist.description}` + : ''; + const url = playlist.external_urls?.spotify ?? ''; + + return { + content: [ + { + type: 'text', + text: + `# Playlist: "${playlist.name}"\n\n` + + `**Owner**: ${owner}\n` + + `**Tracks**: ${tracksTotal}\n` + + `**Visibility**: ${isPublic}${isCollaborative}` + + `${description}\n` + + `**ID**: ${playlist.id}\n` + + `**URL**: ${url}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting playlist: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const updatePlaylist: tool<{ + playlistId: z.ZodString; + name: z.ZodOptional; + description: z.ZodOptional; + public: z.ZodOptional; + collaborative: z.ZodOptional; +}> = { + name: 'updatePlaylist', + description: + 'Update the details of a Spotify playlist (name, description, public/private, collaborative)', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + name: z.string().optional().describe('New name for the playlist'), + description: z + .string() + .optional() + .describe('New description for the playlist'), + public: z + .boolean() + .optional() + .describe('Whether the playlist should be public'), + collaborative: z + .boolean() + .optional() + .describe( + 'Whether the playlist should be collaborative (requires public to be false)', + ), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { + playlistId, + name, + description, + public: isPublic, + collaborative, + } = args; + + if ( + !name && + description === undefined && + isPublic === undefined && + collaborative === undefined + ) { + return { + content: [ + { + type: 'text', + text: 'Error: At least one field to update must be provided (name, description, public, collaborative)', + }, + ], + }; + } + + try { + const body: Record = {}; + if (name) body.name = name; + if (description !== undefined) body.description = description; + if (isPublic !== undefined) body.public = isPublic; + if (collaborative !== undefined) body.collaborative = collaborative; + + await handleSpotifyRequest(async (spotifyApi) => { + await spotifyApi.playlists.changePlaylistDetails(playlistId, body); + }); + + const changes = Object.keys(body).join(', '); + return { + content: [ + { + type: 'text', + text: `Successfully updated playlist (ID: ${playlistId})\nFields updated: ${changes}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error updating playlist: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const removeTracksFromPlaylist: tool<{ + playlistId: z.ZodString; + trackIds: z.ZodArray; + snapshotId: z.ZodOptional; +}> = { + name: 'removeTracksFromPlaylist', + description: + 'Remove one or more tracks from a Spotify playlist (max 100 tracks per request)', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + trackIds: z + .array(z.string()) + .min(1) + .max(100) + .describe('Array of Spotify track IDs to remove (max 100)'), + snapshotId: z + .string() + .optional() + .describe( + 'The playlist snapshot ID to target a specific version (optional)', + ), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { playlistId, trackIds, snapshotId } = args; + + try { + const items = trackIds.map((id) => ({ uri: `spotify:track:${id}` })); + + // Hit /items directly: SDK targets the deprecated /tracks endpoint + // (see spotifyFetch JSDoc for context on the March 2026 migration). + await spotifyFetch(`playlists/${playlistId}/items`, { + method: 'DELETE', + body: { + items, + ...(snapshotId ? { snapshot_id: snapshotId } : {}), + }, + }); + + return { + content: [ + { + type: 'text', + text: `Successfully removed ${trackIds.length} track${ + trackIds.length === 1 ? '' : 's' + } from playlist (ID: ${playlistId})`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error removing tracks from playlist: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const reorderPlaylistItems: tool<{ + playlistId: z.ZodString; + rangeStart: z.ZodNumber; + insertBefore: z.ZodNumber; + rangeLength: z.ZodOptional; + snapshotId: z.ZodOptional; +}> = { + name: 'reorderPlaylistItems', + description: + 'Reorder a range of tracks within a Spotify playlist by moving them to a new position', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + rangeStart: z + .number() + .nonnegative() + .describe('The position of the first item to move (0-based index)'), + insertBefore: z + .number() + .nonnegative() + .describe( + 'The position where the items should be inserted (0-based index)', + ), + rangeLength: z + .number() + .min(1) + .optional() + .describe('Number of consecutive items to move (defaults to 1)'), + snapshotId: z + .string() + .optional() + .describe( + 'The playlist snapshot ID to target a specific version (optional)', + ), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { playlistId, rangeStart, insertBefore, rangeLength, snapshotId } = + args; + + try { + // Hit /items directly: see spotifyFetch JSDoc for context. + await spotifyFetch(`playlists/${playlistId}/items`, { + method: 'PUT', + body: { + range_start: rangeStart, + insert_before: insertBefore, + ...(rangeLength !== undefined ? { range_length: rangeLength } : {}), + ...(snapshotId ? { snapshot_id: snapshotId } : {}), + }, + }); + + const count = rangeLength ?? 1; + return { + content: [ + { + type: 'text', + text: `Successfully moved ${count} track${ + count === 1 ? '' : 's' + } from position ${rangeStart} to before position ${insertBefore} in playlist (ID: ${playlistId})`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error reordering playlist items: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +export const playlistTools = [ + getPlaylist, + updatePlaylist, + removeTracksFromPlaylist, + reorderPlaylistItems, +]; diff --git a/spotify-mcp-server/src/read.ts b/spotify-mcp-server/src/read.ts new file mode 100644 index 0000000..2924286 --- /dev/null +++ b/spotify-mcp-server/src/read.ts @@ -0,0 +1,766 @@ +import type { MaxInt } from '@spotify/web-api-ts-sdk'; +import { z } from 'zod'; +import type { SpotifyHandlerExtra, SpotifyTrack, tool } from './types.js'; +import { + formatDuration, + handleSpotifyRequest, + spotifyFetch, +} from './utils.js'; + +function isTrack(item: any): item is SpotifyTrack { + return ( + item && + item.type === 'track' && + Array.isArray(item.artists) && + item.album && + typeof item.album.name === 'string' + ); +} + +const searchSpotify: tool<{ + query: z.ZodString; + type: z.ZodEnum<['track', 'album', 'artist', 'playlist']>; + limit: z.ZodOptional; +}> = { + name: 'searchSpotify', + description: 'Search for tracks, albums, artists, or playlists on Spotify', + schema: { + query: z.string().describe('The search query'), + type: z + .enum(['track', 'album', 'artist', 'playlist']) + .describe( + 'The type of item to search for either track, album, artist, or playlist', + ), + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of results to return (10-50)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { query, type, limit } = args; + const limitValue = limit ?? 10; + + try { + const results = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.search( + query, + [type], + undefined, + limitValue as MaxInt<50>, + ); + }); + + let formattedResults = ''; + + if (type === 'track' && results.tracks) { + formattedResults = results.tracks.items + .map((track, i) => { + const artists = track.artists.map((a) => a.name).join(', '); + const duration = formatDuration(track.duration_ms); + return `${i + 1}. "${ + track.name + }" by ${artists} (${duration}) - ID: ${track.id}`; + }) + .join('\n'); + } else if (type === 'album' && results.albums) { + formattedResults = results.albums.items + .map((album, i) => { + const artists = album.artists.map((a) => a.name).join(', '); + return `${i + 1}. "${album.name}" by ${artists} - ID: ${album.id}`; + }) + .join('\n'); + } else if (type === 'artist' && results.artists) { + formattedResults = results.artists.items + .map((artist, i) => { + return `${i + 1}. ${artist.name} - ID: ${artist.id}`; + }) + .join('\n'); + } else if (type === 'playlist' && results.playlists) { + formattedResults = results.playlists.items + .map((playlist, i) => { + return `${i + 1}. "${playlist?.name ?? 'Unknown Playlist'} (${ + playlist?.description ?? 'No description' + } tracks)" by ${playlist?.owner?.display_name} - ID: ${ + playlist?.id + }`; + }) + .join('\n'); + } + + return { + content: [ + { + type: 'text', + text: + formattedResults.length > 0 + ? `# Search results for "${query}" (type: ${type})\n\n${formattedResults}` + : `No ${type} results found for "${query}"`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error searching for ${type}s: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getNowPlaying: tool> = { + name: 'getNowPlaying', + description: + 'Get information about the currently playing track on Spotify, including device and volume info', + schema: {}, + handler: async (_args, _extra: SpotifyHandlerExtra) => { + try { + const playback = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.player.getPlaybackState(); + }); + + if (!playback?.item) { + return { + content: [ + { + type: 'text', + text: 'Nothing is currently playing on Spotify', + }, + ], + }; + } + + const item = playback.item; + + if (!isTrack(item)) { + return { + content: [ + { + type: 'text', + text: 'Currently playing item is not a track (might be a podcast episode)', + }, + ], + }; + } + + const artists = item.artists.map((a) => a.name).join(', '); + const album = item.album.name; + const duration = formatDuration(item.duration_ms); + const progress = formatDuration(playback.progress_ms || 0); + const isPlaying = playback.is_playing; + + const device = playback.device; + const deviceInfo = device + ? `${device.name} (${device.type})` + : 'Unknown device'; + const volume = + device?.volume_percent !== null && device?.volume_percent !== undefined + ? `${device.volume_percent}%` + : 'N/A'; + const shuffle = playback.shuffle_state ? 'On' : 'Off'; + const repeat = playback.repeat_state || 'off'; + + return { + content: [ + { + type: 'text', + text: + `# Currently ${isPlaying ? 'Playing' : 'Paused'}\n\n` + + `**Track**: "${item.name}"\n` + + `**Artist**: ${artists}\n` + + `**Album**: ${album}\n` + + `**Progress**: ${progress} / ${duration}\n` + + `**ID**: ${item.id}\n\n` + + `**Device**: ${deviceInfo}\n` + + `**Volume**: ${volume}\n` + + `**Shuffle**: ${shuffle} | **Repeat**: ${repeat}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting current track: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getMyPlaylists: tool<{ + limit: z.ZodOptional; +}> = { + name: 'getMyPlaylists', + description: "Get a list of the current user's playlists on Spotify", + schema: { + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of playlists to return (1-50)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { limit = 50 } = args; + + try { + const playlists = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.currentUser.playlists.playlists( + limit as MaxInt<50>, + ); + }); + + if (playlists.items.length === 0) { + return { + content: [ + { + type: 'text', + text: "You don't have any playlists on Spotify", + }, + ], + }; + } + + const formattedPlaylists = playlists.items + .map((playlist, i) => { + const tracksTotal = playlist.tracks?.total ? playlist.tracks.total : 0; + return `${i + 1}. "${playlist.name}" (${tracksTotal} tracks) - ID: ${ + playlist.id + }`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Your Spotify Playlists\n\n${formattedPlaylists}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting playlists: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getPlaylistTracks: tool<{ + playlistId: z.ZodString; + limit: z.ZodOptional; + offset: z.ZodOptional; +}> = { + name: 'getPlaylistTracks', + description: 'Get a list of tracks in a Spotify playlist', + schema: { + playlistId: z.string().describe('The Spotify ID of the playlist'), + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of tracks to return (1-50)'), + offset: z + .number() + .min(0) + .optional() + .describe('Offset for pagination (0-based index)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { playlistId, limit = 50, offset = 0 } = args; + + // Hit /items directly: see spotifyFetch JSDoc for context. + // Response wraps each entry's track under `item` (new) or `track` (legacy). + type PlaylistItemEntry = { + item?: SpotifyTrack | null; + track?: SpotifyTrack | null; + }; + const playlistTracks = await spotifyFetch<{ + items: PlaylistItemEntry[]; + total: number; + }>(`playlists/${playlistId}/items`, { + query: { limit, offset }, + }); + + if ((playlistTracks.items?.length ?? 0) === 0) { + return { + content: [ + { + type: 'text', + text: "This playlist doesn't have any tracks", + }, + ], + }; + } + + const formattedTracks = playlistTracks.items + .map((entry, i) => { + const track = entry.item ?? entry.track; + if (!track) return `${offset + i + 1}. [Removed track]`; + + if (isTrack(track)) { + const artists = track.artists.map((a) => a.name).join(', '); + const duration = formatDuration(track.duration_ms); + return `${offset + i + 1}. "${track.name}" by ${artists} (${duration}) - ID: ${track.id}`; + } + + return `${offset + i + 1}. Unknown item`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Tracks in Playlist (${offset + 1}-${offset + playlistTracks.items.length} of ${playlistTracks.total})\n\n${formattedTracks}`, + }, + ], + }; + }, +}; + +const getRecentlyPlayed: tool<{ + limit: z.ZodOptional; +}> = { + name: 'getRecentlyPlayed', + description: 'Get a list of recently played tracks on Spotify', + schema: { + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of tracks to return (1-50)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { limit = 50 } = args; + + try { + const history = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.player.getRecentlyPlayedTracks( + limit as MaxInt<50>, + ); + }); + + if (history.items.length === 0) { + return { + content: [ + { + type: 'text', + text: "You don't have any recently played tracks on Spotify", + }, + ], + }; + } + + const formattedHistory = history.items + .map((item, i) => { + const track = item.track; + if (!track) return `${i + 1}. [Removed track]`; + + if (isTrack(track)) { + const artists = track.artists.map((a) => a.name).join(', '); + const duration = formatDuration(track.duration_ms); + const playedAt = item.played_at + ? new Date(item.played_at).toLocaleString() + : 'Unknown time'; + return `${i + 1}. "${track.name}" by ${artists} (${duration}) - ID: ${track.id} - Played at: ${playedAt}`; + } + + return `${i + 1}. Unknown item`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Recently Played Tracks\n\n${formattedHistory}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting recently played tracks: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getUsersSavedTracks: tool<{ + limit: z.ZodOptional; + offset: z.ZodOptional; +}> = { + name: 'getUsersSavedTracks', + description: + 'Get a list of tracks saved in the user\'s "Liked Songs" library', + schema: { + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of tracks to return (1-50)'), + offset: z + .number() + .min(0) + .optional() + .describe('Offset for pagination (0-based index)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { limit = 50, offset = 0 } = args; + + try { + const savedTracks = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.currentUser.tracks.savedTracks( + limit as MaxInt<50>, + offset, + ); + }); + + if (savedTracks.items.length === 0) { + return { + content: [ + { + type: 'text', + text: "You don't have any saved tracks in your Liked Songs", + }, + ], + }; + } + + const formattedTracks = savedTracks.items + .map((item, i) => { + const track = item.track; + if (!track) return `${i + 1}. [Removed track]`; + + if (isTrack(track)) { + const artists = track.artists.map((a) => a.name).join(', '); + const duration = formatDuration(track.duration_ms); + const addedDate = new Date(item.added_at).toLocaleDateString(); + return `${offset + i + 1}. "${track.name}" by ${artists} (${duration}) - ID: ${track.id} - Added: ${addedDate}`; + } + + return `${i + 1}. Unknown item`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `# Your Liked Songs (${offset + 1}-${offset + savedTracks.items.length} of ${savedTracks.total})\n\n${formattedTracks}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting saved tracks: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getQueue: tool<{ + limit: z.ZodOptional; +}> = { + name: 'getQueue', + description: + 'Get a list of the currently playing track and the next items in your Spotify queue', + schema: { + limit: z + .number() + .min(1) + .max(50) + .optional() + .describe('Maximum number of upcoming items to show (1-50)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { limit = 10 } = args; + + try { + const queue = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.player.getUsersQueue(); + }); + + const current = (queue as any)?.currently_playing; + const upcoming = ((queue as any)?.queue ?? []) as any[]; + + const header = '# Spotify Queue\n\n'; + + let currentText = 'Nothing is currently playing'; + if (current) { + const name = current?.name ?? 'Unknown'; + const artists = Array.isArray(current?.artists) + ? (current.artists as Array<{ name: string }>) + .map((a) => a.name) + .join(', ') + : 'Unknown'; + const duration = + typeof current?.duration_ms === 'number' + ? formatDuration(current.duration_ms) + : 'Unknown'; + currentText = `Currently Playing: "${name}" by ${artists} (${duration})`; + } + + if (upcoming.length === 0) { + return { + content: [ + { + type: 'text', + text: `${header}${currentText}\n\nNo upcoming items in the queue`, + }, + ], + }; + } + + const toShow = upcoming.slice(0, limit); + const formatted = toShow + .map((track, i) => { + const name = track?.name ?? 'Unknown'; + const artists = Array.isArray(track?.artists) + ? (track.artists as Array<{ name: string }>) + .map((a) => a.name) + .join(', ') + : 'Unknown'; + const duration = + typeof track?.duration_ms === 'number' + ? formatDuration(track.duration_ms) + : 'Unknown'; + const id = track?.id ?? 'Unknown'; + return `${i + 1}. "${name}" by ${artists} (${duration}) - ID: ${id}`; + }) + .join('\n'); + + return { + content: [ + { + type: 'text', + text: `${header}${currentText}\n\nNext ${toShow.length} in queue:\n\n${formatted}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error fetching queue: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const getAvailableDevices: tool> = { + name: 'getAvailableDevices', + description: + "Get information about the user's available Spotify Connect devices", + schema: {}, + handler: async (_args, _extra: SpotifyHandlerExtra) => { + try { + const devices = await handleSpotifyRequest(async (spotifyApi) => { + return await spotifyApi.player.getAvailableDevices(); + }); + + if (!devices.devices || devices.devices.length === 0) { + return { + content: [ + { + type: 'text', + text: 'No available devices found. Make sure Spotify is open on at least one device.', + }, + ], + }; + } + + const formattedDevices = devices.devices + .map((device, i) => { + const status = device.is_active ? '▶ Active' : '○ Inactive'; + const volume = + device.volume_percent !== null + ? `${device.volume_percent}%` + : 'N/A'; + const restricted = device.is_restricted ? ' (Restricted)' : ''; + return `${i + 1}. ${device.name} (${device.type})${restricted}\n Status: ${status} | Volume: ${volume} | ID: ${device.id}`; + }) + .join('\n\n'); + + return { + content: [ + { + type: 'text', + text: `# Available Spotify Devices\n\n${formattedDevices}`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error getting available devices: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const saveUsersTracks: tool<{ + trackIds: z.ZodArray; +}> = { + name: 'saveUsersTracks', + description: + 'Save (like) one or more tracks to the user\'s "Liked Songs" library (max 50 per request)', + schema: { + trackIds: z + .array(z.string()) + .max(50) + .describe('Array of Spotify track IDs to save/like (max 50)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { trackIds } = args; + + if (trackIds.length === 0) { + return { + content: [{ type: 'text', text: 'Error: No track IDs provided' }], + }; + } + + try { + // PUT /me/tracks with body {"ids": [...]} + await spotifyFetch('me/tracks', { + method: 'PUT', + body: { ids: trackIds }, + }); + + return { + content: [ + { + type: 'text', + text: `Successfully saved ${trackIds.length} track${ + trackIds.length === 1 ? '' : 's' + } to your Liked Songs`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error saving tracks to Liked Songs: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +const removeUsersSavedTracks: tool<{ + trackIds: z.ZodArray; +}> = { + name: 'removeUsersSavedTracks', + description: + 'Remove one or more tracks from the user\'s "Liked Songs" library (max 40 per request)', + schema: { + trackIds: z + .array(z.string()) + .max(40) + .describe('Array of Spotify track IDs to remove (max 40)'), + }, + handler: async (args, _extra: SpotifyHandlerExtra) => { + const { trackIds } = args; + + if (trackIds.length === 0) { + return { + content: [{ type: 'text', text: 'Error: No track IDs provided' }], + }; + } + + try { + // DELETE /me/tracks with body {"ids": [...]} (plain track IDs, not URIs) + await spotifyFetch('me/tracks', { + method: 'DELETE', + body: { ids: trackIds }, + }); + + return { + content: [ + { + type: 'text', + text: `Successfully removed ${trackIds.length} track${ + trackIds.length === 1 ? '' : 's' + } from your Liked Songs`, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error removing tracks from Liked Songs: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + }; + } + }, +}; + +export const readTools = [ + searchSpotify, + getNowPlaying, + getMyPlaylists, + getPlaylistTracks, + getRecentlyPlayed, + getUsersSavedTracks, + saveUsersTracks, + removeUsersSavedTracks, + getQueue, + getAvailableDevices, +]; diff --git a/spotify-mcp-server/src/types.ts b/spotify-mcp-server/src/types.ts new file mode 100644 index 0000000..977e90d --- /dev/null +++ b/spotify-mcp-server/src/types.ts @@ -0,0 +1,53 @@ +import type { RequestHandlerExtra } from '@modelcontextprotocol/sdk/shared/protocol.js'; +import type { + ServerNotification, + ServerRequest, +} from '@modelcontextprotocol/sdk/types.js'; +import type { z } from 'zod'; + +export type SpotifyHandlerExtra = RequestHandlerExtra< + ServerRequest, + ServerNotification +>; + +export type tool = { + name: string; + description: string; + schema: Args; + handler: ( + args: z.infer>, + extra: SpotifyHandlerExtra, + ) => + | Promise<{ + content: Array<{ + type: 'text'; + text: string; + }>; + }> + | { + content: Array<{ + type: 'text'; + text: string; + }>; + }; +}; + +export interface SpotifyArtist { + id: string; + name: string; +} + +export interface SpotifyAlbum { + id: string; + name: string; + artists: SpotifyArtist[]; +} + +export interface SpotifyTrack { + id: string; + name: string; + type: string; + duration_ms: number; + artists: SpotifyArtist[]; + album: SpotifyAlbum; +} diff --git a/spotify-mcp-server/src/utils.ts b/spotify-mcp-server/src/utils.ts new file mode 100644 index 0000000..aa55d35 --- /dev/null +++ b/spotify-mcp-server/src/utils.ts @@ -0,0 +1,439 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import { URL, fileURLToPath } from 'node:url'; +import { SpotifyApi } from '@spotify/web-api-ts-sdk'; +import open from 'open'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const CONFIG_FILE = path.join(__dirname, '../spotify-config.json'); + +export interface SpotifyConfig { + clientId: string; + clientSecret: string; + redirectUri: string; + accessToken?: string; + refreshToken?: string; + expiresAt?: number; // Unix timestamp in milliseconds +} + +export function loadSpotifyConfig(): SpotifyConfig { + if (!fs.existsSync(CONFIG_FILE)) { + throw new Error( + `Spotify configuration file not found at ${CONFIG_FILE}. Please create one with clientId, clientSecret, and redirectUri.`, + ); + } + + try { + const config = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); + if (!(config.clientId && config.clientSecret && config.redirectUri)) { + throw new Error( + 'Spotify configuration must include clientId, clientSecret, and redirectUri.', + ); + } + return config; + } catch (error) { + throw new Error( + `Failed to parse Spotify configuration: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +export function saveSpotifyConfig(config: SpotifyConfig): void { + fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2), 'utf8'); +} + +let cachedSpotifyApi: SpotifyApi | null = null; + +/** + * Direct Spotify Web API fetch helper. + * Used to bypass @spotify/web-api-ts-sdk methods that hit deprecated endpoints + * (e.g. /playlists/{id}/tracks which was retired in the March 2026 API migration + * for new Development Mode apps; replacement is /playlists/{id}/items). + * + * Handles token loading and refresh transparently. + */ +export async function spotifyFetch( + endpoint: string, + options: { + method?: 'GET' | 'POST' | 'PUT' | 'DELETE'; + body?: unknown; + query?: Record; + } = {}, +): Promise { + const { method = 'GET', body, query } = options; + const config = loadSpotifyConfig(); + + // Refresh token if expired + if (config.accessToken && config.refreshToken) { + const now = Date.now(); + if (!config.expiresAt || config.expiresAt <= now) { + const tokens = await refreshAccessToken(config); + config.accessToken = tokens.access_token; + config.expiresAt = now + tokens.expires_in * 1000; + saveSpotifyConfig(config); + cachedSpotifyApi = null; + } + } + + if (!config.accessToken) { + throw new Error( + 'No access token available. Run "npm run auth" to authenticate.', + ); + } + + // Build URL with query string + const cleanEndpoint = endpoint.startsWith('/') ? endpoint.slice(1) : endpoint; + let url = `https://api.spotify.com/v1/${cleanEndpoint}`; + if (query) { + const qs = new URLSearchParams(); + for (const [k, v] of Object.entries(query)) { + if (v !== undefined && v !== null) qs.append(k, String(v)); + } + const qsStr = qs.toString(); + if (qsStr) url += `?${qsStr}`; + } + + const response = await fetch(url, { + method, + headers: { + Authorization: `Bearer ${config.accessToken}`, + ...(body !== undefined ? { 'Content-Type': 'application/json' } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + + if (!response.ok) { + const errBody = await response.text(); + throw new Error( + `Spotify API ${method} ${url} failed (${response.status}): ${errBody}`, + ); + } + + // Some endpoints (DELETE, PUT) return empty body + const text = await response.text(); + if (!text) return undefined as T; + return JSON.parse(text) as T; +} + +export async function createSpotifyApi(): Promise { + const config = loadSpotifyConfig(); + + if (config.accessToken && config.refreshToken) { + const now = Date.now(); + const shouldRefresh = !config.expiresAt || config.expiresAt <= now; + + if (shouldRefresh) { + console.error( + 'Access token expired or missing expiration time, refreshing...', + ); + try { + const tokens = await refreshAccessToken(config); + config.accessToken = tokens.access_token; + config.expiresAt = now + tokens.expires_in * 1000; // Convert seconds to milliseconds + saveSpotifyConfig(config); + console.error('Access token refreshed successfully'); + + // Clear cached API instance to force recreation with new token + cachedSpotifyApi = null; + } catch (error) { + console.error('Failed to refresh token:', error); + throw new Error( + 'Failed to refresh access token. Please run "npm run auth" to re-authenticate.', + ); + } + } + + if (cachedSpotifyApi) { + return cachedSpotifyApi; + } + + const accessToken = { + access_token: config.accessToken, + token_type: 'Bearer', + expires_in: Math.floor( + ((config.expiresAt ?? now + 3600000) - now) / 1000, + ), + refresh_token: config.refreshToken, + }; + + cachedSpotifyApi = SpotifyApi.withAccessToken(config.clientId, accessToken); + return cachedSpotifyApi; + } + + // Fallback to client credentials if no user tokens available + cachedSpotifyApi = SpotifyApi.withClientCredentials( + config.clientId, + config.clientSecret, + ); + + return cachedSpotifyApi; +} + +function generateRandomString(length: number): string { + const array = new Uint8Array(length); + crypto.getRandomValues(array); + return Array.from(array) + .map((b) => + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'.charAt( + b % 62, + ), + ) + .join(''); +} + +function base64Encode(str: string): string { + return Buffer.from(str).toString('base64'); +} + +async function exchangeCodeForToken( + code: string, + config: SpotifyConfig, +): Promise<{ + access_token: string; + refresh_token: string; + expires_in: number; +}> { + const tokenUrl = 'https://accounts.spotify.com/api/token'; + const authHeader = `Basic ${base64Encode(`${config.clientId}:${config.clientSecret}`)}`; + + const params = new URLSearchParams(); + params.append('grant_type', 'authorization_code'); + params.append('code', code); + params.append('redirect_uri', config.redirectUri); + + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { + Authorization: authHeader, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params, + }); + + if (!response.ok) { + const errorData = await response.text(); + throw new Error(`Failed to exchange code for token: ${errorData}`); + } + + const data = await response.json(); + return { + access_token: data.access_token, + refresh_token: data.refresh_token, + expires_in: data.expires_in || 3600, + }; +} + +async function refreshAccessToken( + config: SpotifyConfig, +): Promise<{ access_token: string; expires_in: number }> { + if (!config.refreshToken) { + throw new Error('No refresh token available'); + } + + const tokenUrl = 'https://accounts.spotify.com/api/token'; + const authHeader = `Basic ${base64Encode(`${config.clientId}:${config.clientSecret}`)}`; + + const params = new URLSearchParams(); + params.append('grant_type', 'refresh_token'); + params.append('refresh_token', config.refreshToken); + + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { + Authorization: authHeader, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params, + }); + + if (!response.ok) { + const errorData = await response.text(); + throw new Error(`Failed to refresh access token: ${errorData}`); + } + + const data = await response.json(); + return { + access_token: data.access_token, + expires_in: data.expires_in || 3600, + }; +} + +export async function authorizeSpotify(): Promise { + const config = loadSpotifyConfig(); + + const redirectUri = new URL(config.redirectUri); + if ( + redirectUri.hostname !== 'localhost' && + redirectUri.hostname !== '127.0.0.1' + ) { + console.error( + 'Error: Redirect URI must use localhost for automatic token exchange', + ); + console.error( + 'Please update your spotify-config.json with a localhost redirect URI', + ); + console.error('Example: http://127.0.0.1:8888/callback'); + process.exit(1); + } + + const port = redirectUri.port || '80'; + const callbackPath = redirectUri.pathname || '/callback'; + + const state = generateRandomString(16); + + const scopes = [ + 'user-read-private', + 'user-read-email', + 'user-read-playback-state', + 'user-modify-playback-state', + 'user-read-currently-playing', + 'playlist-read-private', + 'playlist-read-collaborative', + 'playlist-modify-private', + 'playlist-modify-public', + 'user-library-read', + 'user-library-modify', + 'user-read-recently-played', + ]; + + const authParams = new URLSearchParams({ + client_id: config.clientId, + response_type: 'code', + redirect_uri: config.redirectUri, + scope: scopes.join(' '), + state: state, + show_dialog: 'true', + }); + + const authorizationUrl = `https://accounts.spotify.com/authorize?${authParams.toString()}`; + + const authPromise = new Promise((resolve, reject) => { + // Create HTTP server to handle the callback + const server = http.createServer(async (req, res) => { + if (!req.url) { + return res.end('No URL provided'); + } + + const reqUrl = new URL(req.url, `http://localhost:${port}`); + + if (reqUrl.pathname === callbackPath) { + const code = reqUrl.searchParams.get('code'); + const returnedState = reqUrl.searchParams.get('state'); + const error = reqUrl.searchParams.get('error'); + + res.writeHead(200, { 'Content-Type': 'text/html' }); + + if (error) { + console.error(`Authorization error: ${error}`); + res.end( + '

Authentication Failed

Please close this window and try again.

', + ); + server.close(); + reject(new Error(`Authorization failed: ${error}`)); + return; + } + + if (returnedState !== state) { + console.error('State mismatch error'); + res.end( + '

Authentication Failed

State verification failed. Please close this window and try again.

', + ); + server.close(); + reject(new Error('State mismatch')); + return; + } + + if (!code) { + console.error('No authorization code received'); + res.end( + '

Authentication Failed

No authorization code received. Please close this window and try again.

', + ); + server.close(); + reject(new Error('No authorization code received')); + return; + } + + try { + const tokens = await exchangeCodeForToken(code, config); + + config.accessToken = tokens.access_token; + config.refreshToken = tokens.refresh_token; + config.expiresAt = Date.now() + tokens.expires_in * 1000; // Convert seconds to milliseconds + saveSpotifyConfig(config); + + res.end( + '

Authentication Successful!

You can now close this window and return to the application.

', + ); + console.log( + 'Authentication successful! Access token has been saved.', + ); + + server.close(); + resolve(); + } catch (error) { + console.error('Token exchange error:', error); + res.end( + '

Authentication Failed

Failed to exchange authorization code for tokens. Please close this window and try again.

', + ); + server.close(); + reject(error); + } + } else { + res.writeHead(404); + res.end(); + } + }); + + server.listen(Number.parseInt(port), '127.0.0.1', () => { + console.log( + `Listening for Spotify authentication callback on port ${port}`, + ); + console.log('Opening browser for authorization...'); + + open(authorizationUrl).catch((_error: Error) => { + console.log( + 'Failed to open browser automatically. Please visit this URL to authorize:', + ); + console.log(authorizationUrl); + }); + }); + + server.on('error', (error) => { + console.error(`Server error: ${error.message}`); + reject(error); + }); + }); + + await authPromise; +} + +export function formatDuration(ms: number): string { + const minutes = Math.floor(ms / 60000); + const seconds = ((ms % 60000) / 1000).toFixed(0); + return `${minutes}:${seconds.padStart(2, '0')}`; +} + +export async function handleSpotifyRequest( + action: (spotifyApi: SpotifyApi) => Promise, +): Promise { + try { + const spotifyApi = await createSpotifyApi(); + return await action(spotifyApi); + } catch (error) { + // Skip JSON parsing errors as these are actually successful operations + const errorMessage = error instanceof Error ? error.message : String(error); + if ( + errorMessage.includes('Unexpected token') || + errorMessage.includes('Unexpected non-whitespace character') || + errorMessage.includes('Exponent part is missing a number in JSON') + ) { + return undefined as T; + } + // Rethrow other errors + throw error; + } +} diff --git a/spotify-mcp-server/tsconfig.json b/spotify-mcp-server/tsconfig.json new file mode 100644 index 0000000..a14bee0 --- /dev/null +++ b/spotify-mcp-server/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Node16", + "moduleResolution": "Node16", + "outDir": "./build", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules"] +} diff --git a/spotify-mcp-server/verify_tools.mjs b/spotify-mcp-server/verify_tools.mjs new file mode 100644 index 0000000..3774021 --- /dev/null +++ b/spotify-mcp-server/verify_tools.mjs @@ -0,0 +1,171 @@ +// Simple MCP Server Tool Verification +import { spawn } from 'child_process'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +console.log('='.repeat(60)); +console.log('SPOTIFY MCP SERVER TOOL VERIFICATION'); +console.log('='.repeat(60) + '\n'); + +// Start server +let server = spawn('node', ['build/index.js'], { + cwd: __dirname, + stdio: ['pipe', 'pipe', 'pipe'] +}); + +const pendingRequests = new Map(); +let requestIdCounter = 1; + +// Track results +const results = []; + +server.stdout.on('data', (data) => { + const msg = data.toString(); + + // Parse each line as JSON response + for (const line of msg.split('\n')) { + if (!line.trim()) continue; + + try { + const response = JSON.parse(line); + const id = response.id; + + // Handle initialization response + if (id === 1) { + if (response.result?.protocolVersion) { + console.log('✅ MCP Server Initialized'); + console.log(` Protocol: ${response.result.protocolVersion}`); + console.log(''); + + // Start tests after a delay + setTimeout(runTests, 300); + } else if (response.error) { + console.log('❌ Initialization failed:', response.error.message); + process.exit(1); + } + continue; + } + + // Handle tool responses + if (pendingRequests.has(id)) { + const req = pendingRequests.get(id); + + if (response.error) { + console.log(`❌ ${req.tool}: ERROR`); + results.push({ name: req.tool, success: false }); + req.resolve(); + } else if (response.result?.content) { + const text = response.result.content[0]?.text || ''; + console.log(`✅ ${req.tool}: OK`); + results.push({ name: req.tool, success: true }); + req.resolve(); + } else { + console.log(`⚠️ ${req.tool}: Unknown response`); + results.push({ name: req.tool, success: false }); + req.resolve(); + } + + pendingRequests.delete(id); + } + } catch (e) { + // Not JSON or parse error - skip + } + } +}); + +server.stderr.on('data', () => {}); + +// Send MCP request helper +function sendRequest(toolName, args) { + return new Promise(resolve => { + const id = requestIdCounter++; + pendingRequests.set(id, { tool: toolName, resolve }); + + const request = JSON.stringify({ + jsonrpc: "2.0", + id: id, + method: "tools/call", + params: { name: toolName, arguments: args } + }) + '\n'; + + server.stdin.write(request); + }); +} + +// Send initialization request after a short delay to ensure stdin is ready +setTimeout(() => { + const initRequest = JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { name: "verify-tools", version: "1.0" } + } + }) + '\n'; + + server.stdin.write(initRequest); +}, 100); + +// Run all tests +async function runTests() { + console.log('Testing MCP Server Tools...\n'); + console.log('-'.repeat(60)); + + // ===== READ OPERATIONS ===== + await sendRequest('searchSpotify', { query: 'rock', type: 'track', limit: 1 }); + await new Promise(r => setTimeout(r, 300)); + + await sendRequest('getNowPlaying', {}); + await new Promise(r => setTimeout(r, 300)); + + await sendRequest('getMyPlaylists', { limit: 2 }); + await new Promise(r => setTimeout(r, 300)); + + await sendRequest('getQueue', {}); + await new Promise(r => setTimeout(r, 300)); + + await sendRequest('getAvailableDevices', {}); + await new Promise(r => setTimeout(r, 300)); + + // ===== PLAYBACK CONTROL ===== + await sendRequest('pausePlayback', {}); + await new Promise(r => setTimeout(r, 300)); + + await sendRequest('resumePlayback', {}); + await new Promise(r => setTimeout(r, 300)); + + // ===== SEARCH VARIATIONS ===== + await sendRequest('searchSpotify', { query: 'beatles', type: 'artist', limit: 1 }); + await new Promise(r => setTimeout(r, 300)); + + // Print summary and exit after delay + setTimeout(() => { + console.log('\n' + '='.repeat(60)); + console.log('VERIFICATION COMPLETE'); + console.log('='.repeat(60) + '\n'); + + const successCount = results.filter(r => r.success).length; + const failCount = results.length - successCount; + + console.log(`Total Tests: ${results.length}`); + console.log(`✅ Passed: ${successCount}`); + console.log(`❌ Failed: ${failCount}\n`); + + if (server) server.kill(); + + console.log('\nTools Verified:'); + console.log('- searchSpotify (search tracks, artists)'); + console.log('- getNowPlaying (current playback state)'); + console.log('- getMyPlaylists (user playlists)'); + console.log('- getQueue (playback queue)'); + console.log('- getAvailableDevices (Spotify Connect devices)'); + console.log('- pausePlayback / resumePlayback'); + + process.exit(0); + }, 2000); +} \ No newline at end of file diff --git a/src/javis/__init__.py b/src/javis/__init__.py new file mode 100644 index 0000000..8c7a1de --- /dev/null +++ b/src/javis/__init__.py @@ -0,0 +1,2 @@ +"""Javis — local voice AI agent.""" +__version__ = "0.1.0" diff --git a/src/javis/__main__.py b/src/javis/__main__.py new file mode 100644 index 0000000..b6f1773 --- /dev/null +++ b/src/javis/__main__.py @@ -0,0 +1,98 @@ +"""Javis entry point.""" +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(prog="javis", description="Local voice AI agent") + p.add_argument("--config", type=Path, default=Path("config.yaml"), + help="Path to config.yaml (default: ./config.yaml)") + p.add_argument("--headless", action="store_true", help="Run without GUI") + p.add_argument("--doctor", action="store_true", help="Check environment and exit") + p.add_argument("--install-deps", action="store_true", + help="Install missing dependencies based on detected hardware and exit") + p.add_argument("--repair", action="store_true", + help="Reinstall PyTorch matching current hardware (fixes DLL errors) and exit") + p.add_argument("--skip-preflight", action="store_true", + help="Skip the startup environment preflight check") + p.add_argument("--speak", type=str, default=None, help="Synthesize a phrase and exit") + p.add_argument("--voice", type=str, default=None, help="Voice profile name (for --speak)") + p.add_argument("--list-devices", action="store_true", help="List audio devices and exit") + return p.parse_args() + + +def _preflight(log) -> None: + """Lightweight checks that print actionable hints before subsystems init. + + Does not abort startup — the orchestrator has fallbacks (NullVAD etc.) + so the user can still get a usable agent on a partially broken env. + """ + from javis.core.hardware import check_torch_loadable, detect_hardware, format_remediation + + ok, msg = check_torch_loadable() + if ok: + return + info = detect_hardware() + log.warning("Preflight: PyTorch is not loadable — VAD will run in fallback mode.") + log.warning(f" reason: {msg}") + for line in format_remediation(info, torch_error=msg).splitlines(): + log.warning(" " + line) + + +def main() -> int: + args = _parse_args() + + # Bootstrap flags run before config/logging so they work on a half-broken env. + if args.install_deps: + from javis.core.bootstrap import run_install + return run_install() + if args.repair: + from javis.core.bootstrap import run_repair + return run_repair() + + from javis.core.config import load_config + from javis.core.logger import setup_logger, get_logger + + config = load_config(args.config) + setup_logger(config) + log = get_logger("main") + log.info(f"Javis starting. config={args.config}") + + if not args.skip_preflight: + try: + _preflight(log) + except Exception as e: # noqa: BLE001 + log.warning(f"preflight check failed: {e}") + + if args.list_devices: + from javis.audio_io.devices import print_devices + print_devices() + return 0 + + if args.doctor: + from javis.core.doctor import run_doctor + return run_doctor(config) + + if args.speak is not None: + from javis.tts.runner import speak_once + return asyncio.run(speak_once(config, args.speak, args.voice)) + + if args.headless: + from javis.agent.orchestrator import run_headless + try: + asyncio.run(run_headless(config)) + except KeyboardInterrupt: + log.info("Interrupted.") + return 0 + + # GUI mode + from javis.gui.main_window import run_gui + return run_gui(config) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/javis/agent/__init__.py b/src/javis/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/javis/agent/middleware.py b/src/javis/agent/middleware.py new file mode 100644 index 0000000..efb8bcc --- /dev/null +++ b/src/javis/agent/middleware.py @@ -0,0 +1,1842 @@ +"""Agentic middleware for Javis. + +Inspired by TanStack AI's middleware architecture — plain objects with optional +lifecycle hooks, composable in array order, firing at key points in the agent loop. + +Composition rules (matching TanStack semantics): + on_before_llm — piped (each sees the output of the previous; return None = pass-through) + on_before_tool — first-win (first non-None decision short-circuits the rest) + on_after_tool — sequential (all run regardless) + on_finish — sequential (all run; use ctx.defer() for non-blocking side effects) + +Lifecycle per turn (maps to TanStack hooks): + on_before_llm(ctx, messages, iteration) → onConfig phase='beforeModel' + on_before_tool(ctx, tool_name, args) → onBeforeToolCall + on_after_tool(ctx, tool_name, args, ...) → onAfterToolCall + on_finish(ctx, user_text, reply) → onFinish + ctx.defer() +""" +from __future__ import annotations + +import asyncio +import json +import re +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union + +from ..core.logger import get_logger + +if TYPE_CHECKING: + from ..core.config import Config + from ..llm.hermes_client import HermesClient + from ..llm.ollama_client import OllamaClient + from ..stt.voice_profiler import SpeakerPrivileges + +log = get_logger("agent.middleware") + +# Strong references to fire-and-forget background tasks. +# asyncio / CPython only hold *weak* references to Tasks — without a strong +# reference the garbage collector can silently cancel a task before it +# completes. We add each task here and remove it via done_callback. +_background_tasks: "set[asyncio.Task]" = set() + + +async def drain_background_tasks() -> None: + """Wait for deferred turn side effects before clients are closed.""" + while _background_tasks: + await asyncio.gather(*list(_background_tasks), return_exceptions=True) + + +# ── Context ─────────────────────────────────────────────────────────────────── + +@dataclass +class TurnContext: + """Request-scoped context available in every hook. + + Analogous to TanStack's ``ChatMiddlewareContext``. One instance is created + per agent turn and threaded through all pipeline calls. + """ + turn_id: str + iteration: int = 0 + _deferred: list = field(default_factory=list, repr=False) + # Compact one-line summaries of every tool call made this turn. + # Populated by MemoryMiddleware.on_after_tool; passed to extraction LLM. + tool_log: list = field(default_factory=list, repr=False) + # Active speaker privileges for this turn. Set by the orchestrator after + # speaker identification; consulted by PrivilegeMiddleware.on_before_tool. + privileges: "Optional[SpeakerPrivileges]" = field(default=None, repr=False) + # Resolved speaker name (or None if identification failed / disabled). + speaker_name: Optional[str] = field(default=None, repr=False) + + def defer(self, coro) -> None: + """Register a fire-and-forget coroutine to run after the turn completes. + + Analogous to TanStack's ``ctx.defer(promise)``. The coroutine is + scheduled with ``asyncio.ensure_future`` when ``flush()`` is called, + so it never blocks the current turn or the next one. + """ + self._deferred.append(coro) + + async def flush(self) -> None: + """Schedule all deferred coroutines and clear the queue. + + Call this at the end of each agent turn after ``run_finish`` has + populated the deferred list. + """ + for coro in self._deferred: + task = asyncio.create_task(coro) + # Keep a strong reference until the task finishes so the GC cannot + # silently cancel it before completion (the event loop only holds + # weak references). + _background_tasks.add(task) + task.add_done_callback(_background_tasks.discard) + self._deferred.clear() + + +# ── BeforeToolCall decisions ────────────────────────────────────────────────── + +@dataclass +class SkipToolDecision: + """Short-circuit the tool — use *result* as if the tool had executed. + + Analogous to TanStack's ``{ type: 'skip', result }``. + """ + result: str + type: str = field(default="skip", init=False) + + +@dataclass +class TransformArgsDecision: + """Replace the tool arguments before execution. + + Analogous to TanStack's ``{ type: 'transformArgs', args }``. + """ + args: dict + type: str = field(default="transformArgs", init=False) + + +@dataclass +class AbortTurnDecision: + """Abort the entire turn immediately. + + Analogous to TanStack's ``{ type: 'abort', reason? }``. + """ + reason: str = "" + type: str = field(default="abort", init=False) + + +BeforeToolDecision = Union[SkipToolDecision, TransformArgsDecision, AbortTurnDecision] + + +# ── Base middleware ─────────────────────────────────────────────────────────── + +class AgentMiddleware: + """Base class with no-op hook defaults. + + Subclass and override only the hooks you need — the same philosophy as + TanStack's plain-object middleware interface. + """ + + name: str = "unnamed" + + # ── piped ───────────────────────────────────────────────────────────────── + + async def on_before_llm( + self, + ctx: TurnContext, + messages: list[dict], + iteration: int, + ) -> Optional[list[dict]]: + """Called before each model call in the agentic loop. + + Return a modified messages list to change what the model sees, or + ``None`` to pass the messages through unchanged. + + Analogous to TanStack ``onConfig phase='beforeModel'``. + """ + return None + + # ── first-win ───────────────────────────────────────────────────────────── + + async def on_before_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + ) -> Optional[BeforeToolDecision]: + """Called before each tool executes. + + Return a ``BeforeToolDecision`` to intercept the call, or ``None`` + to let it proceed normally. The first middleware that returns a + non-``None`` decision short-circuits the rest. + + Analogous to TanStack ``onBeforeToolCall``. + """ + return None + + # ── sequential ──────────────────────────────────────────────────────────── + + async def on_after_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + result: str, + ok: bool, + latency: float, + ) -> None: + """Called after each tool execution (or skip). + + All middleware run in order regardless of the outcome. + + Analogous to TanStack ``onAfterToolCall``. + """ + + async def on_finish( + self, + ctx: TurnContext, + user_text: str, + reply: str, + ) -> None: + """Called once after the turn produces its final reply. + + Use ``ctx.defer(coro)`` for non-blocking side effects that must not + delay the response. + + Analogous to TanStack ``onFinish`` + ``ctx.defer()``. + """ + + async def on_start(self, ctx: TurnContext) -> None: + """Called once at the very start of each agent turn. + + Useful for logging, per-turn counters, or setup that must run before + ``on_before_llm``. Analogous to TanStack ``onStart``. + """ + + async def on_abort(self, ctx: TurnContext, reason: str) -> None: + """Called when the turn is aborted via ``AbortTurnDecision``. + + Exactly one of ``on_finish``, ``on_abort``, or ``on_error`` fires per + turn. Analogous to TanStack ``onAbort``. + """ + + async def on_error(self, ctx: TurnContext, exc: Exception) -> None: + """Called when an unhandled exception terminates the turn. + + Exactly one of ``on_finish``, ``on_abort``, or ``on_error`` fires per + turn. Analogous to TanStack ``onError``. + """ + + +# ── Memory middleware ───────────────────────────────────────────────────────── + +# Maximum characters of the assistant reply fed to the extraction call +# (keeps the context short for the background LLM call). +_EXTRACT_REPLY_CAP = 500 + +# Maximum number of memory entries injected into the system message. +_RECALL_MAX_RESULTS = 8 + +# Minimum word length for keyword matching (avoids stop-word noise). +_RECALL_MIN_WORD_LEN = 4 + +# Common German/English words (≥4 chars) that must never drive keyword recall. +# Without this filter, filler words like "nicht"/"bitte" match nearly every +# stored rule and recall degrades to noise. +_RECALL_STOPWORDS: frozenset[str] = frozenset({ + # German + "aber", "alle", "allem", "allen", "alles", "also", "andere", "auch", + "bevor", "bist", "bitte", "dann", "danke", "dass", "dein", "deine", + "denn", "dich", "diese", "dieser", "dieses", "doch", "dort", "eben", + "eine", "einem", "einen", "einer", "eines", "etwas", "euch", "ganz", + "gerade", "gerne", "geht", "gibt", "habe", "haben", "hast", "hatte", + "hallo", "heute", "hier", "ihre", "ihrer", "immer", "irgendwie", "jetzt", + "kann", "kannst", "kein", "keine", "können", "könnte", "könntest", + "machen", "mache", "machst", "mehr", "mein", "meine", "meinen", "mich", + "muss", "musst", "nach", "nein", "nicht", "nichts", "noch", "nochmal", + "oder", "ohne", "okay", "schon", "sein", "seine", "sind", "soll", + "sollen", "sollst", "sollte", "sowie", "viel", "vielleicht", "vielen", + "wann", "warum", "weil", "welche", "welcher", "welches", "wenn", + "werde", "werden", "wieder", "wieso", "wird", "wirst", "wollen", + "wollte", "würde", "würdest", "zwischen", + # English + "about", "above", "after", "again", "also", "always", "another", + "anything", "been", "before", "being", "both", "cannot", "could", + "does", "doing", "done", "down", "each", "else", "even", "ever", + "every", "everything", "from", "give", "goes", "going", "gone", + "good", "have", "having", "here", "into", "just", "know", "like", + "little", "look", "made", "make", "many", "maybe", "mine", "more", + "most", "much", "must", "need", "never", "next", "nice", "none", + "nothing", "only", "onto", "other", "over", "please", "really", + "right", "said", "same", "should", "show", "since", "some", + "something", "soon", "still", "such", "sure", "take", "tell", "than", + "thank", "thanks", "that", "their", "them", "then", "there", "these", + "they", "thing", "think", "this", "those", "until", "very", "want", + "wants", "well", "were", "what", "when", "where", "which", "while", + "will", "with", "without", "would", "yeah", "your", "yours", +}) + +# Categories injected as standing rules when no keyword matches — behavioural +# rules are always safe context; random recent facts are not. +_FALLBACK_CATEGORIES: frozenset[str] = frozenset({"preferences", "smart_home"}) + +# Background consolidation cadence and safety bounds. +_CONSOLIDATE_EVERY_TURNS = 50 +_CONSOLIDATE_MIN_ENTRIES = 12 +_CONSOLIDATE_MAX_OPS = 10 + +# Maximum characters of the user message sent to the extraction LLM. +_EXTRACT_USER_TEXT_CAP = 300 + +# Memory categories stored in the speaker's personal file. +# Everything else (smart_home, spotify, facts, context) goes to the shared +# household file so that device knowledge is visible to all speakers. +_PERSONAL_CATEGORIES: frozenset[str] = frozenset({"personal", "preferences", "tasks"}) + +_EXTRACTION_SYSTEM_PROMPT = """\ +You are a memory extraction assistant for a bilingual (German/English) general-purpose \ +AI assistant called Javis. Javis handles all kinds of tasks (smart home, music, web \ +search, media, files, questions) via named MCP tools. Your only job is to identify \ +0-{MAX_FACTS} facts from the conversation snippet below that are worth persisting \ +for future sessions. + +DURABILITY TEST -- apply to EVERY candidate fact before extracting: +This is LONG-TERM memory, kept across years of usage. Ask yourself: +"Will this fact still be true and useful in one year?" + * YES -> extract it (identity, preferences, setup, behavioural rules, patterns). + * NO -> do NOT extract it (query results, current states, one-time events). +When a transient query REVEALS a stable underlying fact, extract ONLY the stable +fact, generalized -- never the transient data: + * Weather query mentions the user's town -> extract user_location, NOT the forecast. + * Search result about a topic the user works on -> extract the interest, NOT the answer. + * A played song reveals a stated taste -> extract the taste, NOT the song instance. + +ALWAYS EXTRACT -- highest priority, never skip: + * Explicit memory instructions: + "merke dir", "remember", "don't forget", "ich moechte dass du weisst", + "behalte im Kopf", "speichere", "save that", and any similar phrasing. + * Behavioral corrections / mistakes: + "du hast X gemacht, ich wollte Y", "nicht X sondern Y", + "das war falsch", "you did X wrong", "only lights, not everything". + * Persistent preferences (how the user wants actions executed): + "always ask first", "only class light", "use the zone command", + "never turn off the router", "volume never above 60". + * Personal facts: name, location, household members, preferences, active projects. + * Smart-home setup revealed by usage: + custom device names, zone names, unusual capabilities the user describes. + * Spotify setup: preferred playback device name/ID, preferred playlists or artists. + +When [TOOL CALLS THIS TURN] are present, also extract: + * Lessons from FAILED tool calls: wrong tool chosen, wrong argument format, + invented device ID, wrong endpoint -- frame the lesson as a rule to follow. + * Working patterns: which tool+argument combo successfully accomplished the goal. + Only extract a positive lesson if it was non-obvious (e.g. zone commands work + better than per-device calls for this user's setup). + Do NOT extract a lesson when all tool calls succeeded with no errors and + the user made no correction -- that means the workflow is already established. + +NEVER EXTRACT (fails the durability test): + * Transient query RESULTS of any kind: weather data, news, search answers, + sensor values, prices, track/media info, dates, current states. These are + outdated within hours -- storing them poisons long-term memory. + * One-time events/observations: a specific song played, a film watched, + a question answered, a device state at some moment. + * Simple one-shot commands with no preference or lesson: + "musik stopp", "licht aus", "pause", "skip", "volume up", "timer 5 min". + * General knowledge the LLM already knows (history, definitions, calculations). + * Facts already stored accurately in [EXISTING MEMORIES] -- do NOT duplicate, + not even under a different key or slightly different phrasing. + You may UPDATE an existing key only if the new value meaningfully supersedes it. + * Successful routine tool calls with no error and no user correction -- + the workflow is already established, return []. + * javis__quick_* and javis__timer_* calls -- these are bookkeeping only, not + learnable patterns. + * Values that only make sense relative to NOW ("morgen", "heute", "aktuell", + "tomorrow", "currently") -- long-term memory must be timeless. + +NOTE: The ALWAYS EXTRACT rules override the NEVER EXTRACT rules. +A "merke dir" instruction is ALWAYS worth storing, even if it sounds trivial. +A behavioral correction is ALWAYS worth storing, even after a simple command. + +VALUE FORMAT: +Write values as actionable rules Javis should follow, not as passive observations. + GOOD: "Immer homey__control_zone statt control_all nutzen, wenn der Nutzer ein Zimmer nennt." + BAD: "The user prefers zone commands." +Keep values under 200 characters. Use German for smart-home / behavioral rules; +use English only for technical patterns (tool names, argument keys). +Keys must be snake_case, max 64 chars, descriptive enough to recall without context. + +Response format -- JSON array ONLY, no prose, no markdown fences: +[{"key": "snake_case_key", "value": "actionable rule", "category": "category_name"}] +If nothing new is worth storing: [] + +Categories: personal | preferences | smart_home | spotify | tasks | facts + +EXAMPLES: + +Example 1 -- Explicit memory instruction (always extract): +[USER]: Merke dir bitte: mein Schlafzimmer-Licht heisst in Homey "Nachttisch Florian". +[ASSISTANT]: Gemerkt. +-> [{"key": "homey_device_schlafzimmer_licht", "value": "Schlafzimmer-Licht des Nutzers heisst in Homey 'Nachttisch Florian'.", "category": "smart_home"}] + +Example 2 -- Behavioral correction (always extract): +[USER]: Stopp. Ich wollte nur die Lichter aus, nicht alle Geraete. +[ASSISTANT]: Verstanden, ich schalte nur die Lichter aus. +-> [{"key": "licht_aus_nur_light_klasse", "value": "Bei 'Licht aus'-Befehlen nur Geraete der Klasse 'light' schalten. homey__control_all schaltet alle Klassen und darf dafuer nicht verwendet werden.", "category": "smart_home"}] + +Example 3 -- Failed tool call lesson (extract from tool log): +[TOOL CALLS THIS TURN] + homey__set_capability({"device_id": "abc-123-fake"}) -> ERR: No device with id 'abc-123-fake'. +[USER]: Kuechenlicht an. +[ASSISTANT]: Ich suche das Geraet. +-> [{"key": "homey_device_id_nie_raten", "value": "Homey-Geraete-IDs NIEMALS erfinden. Immer zuerst homey__find_devices aufrufen und die echte UUID aus dem Ergebnis verwenden.", "category": "smart_home"}] + +Example 4 -- Spotify setup fact: +[TOOL CALLS THIS TURN] + getAvailableDevices() -> OK: [{"name": "Florian", "id": "abc123"}, ...] +[USER]: Spiel Musik auf meinem Geraet. +[ASSISTANT]: Musik laeuft auf Florian. +-> [{"key": "spotify_hauptgeraet_name", "value": "Bevorzugtes Spotify-Wiedergabegeraet heisst 'Florian'. Bei Musikbefehlen ohne explizites Zielgeraet dieses Geraet verwenden.", "category": "spotify"}] + +Example 5 -- Simple command, nothing to extract: +[USER]: Musik stopp. +[ASSISTANT]: Wiedergabe pausiert. +-> [] + +Example 9 -- Transient query reveals a stable fact (extract ONLY the stable fact): +[TOOL CALLS THIS TURN] + duckduckgo__ddg_answers({"query": "Wetter Schrobenhausen"}) -> OK: Wetter Schrobenhausen heute ... 80% Niederschlag ... +[USER]: Wie ist das Wetter? +[ASSISTANT]: Es regnet wahrscheinlich, 21 bis 28 Grad. +-> [{"key": "user_location", "value": "Der Nutzer wohnt in Schrobenhausen. Bei ortsbezogenen Anfragen ohne explizite Ortsangabe Schrobenhausen verwenden.", "category": "personal"}] +(NOT the forecast values -- they are outdated within hours.) + +Example 10 -- Transient query result, no stable fact revealed: +[TOOL CALLS THIS TURN] + duckduckgo__ddg_answers({"query": "Oscar Gewinner 2024"}) -> OK: Emma Stone ... +[USER]: Wer hat den Oscar 2024 gewonnen? +[ASSISTANT]: Emma Stone gewann den Oscar als beste Hauptdarstellerin. +-> [] + +Example 7 -- Successful zone control, no correction (never extract): +[TOOL CALLS THIS TURN] + homey__control_zone({"zone": "Wohnzimmer", "capability": "onoff", "value": false}) -> OK: Zone controlled +[USER]: Wohnzimmerlicht aus. +[ASSISTANT]: Wohnzimmer Licht aus. +-> [] + +Example 8 -- Successful zone control WITH user correction (always extract): +[TOOL CALLS THIS TURN] + homey__control_zone({"zone": "Wohnzimmer", "capability": "onoff", "value": false}) -> OK: Zone controlled +[USER]: Wohnzimmerlicht aus. (correction in same session: "Ich wollte nur die Lichter, nicht den Fernseher") +[ASSISTANT]: Verstanden, naechstes Mal nur Lichter. +-> [{"key": "wohnzimmer_licht_aus_nur_lights", "value": "Bei 'Wohnzimmerlicht aus' nur Geraete der Klasse 'light' schalten, nicht andere Geraete wie TV oder Stecker.", "category": "smart_home"}] + +Example 6 -- Update an existing memory: +[EXISTING MEMORIES] +* spotify_hauptgeraet_name: Bevorzugtes Spotify-Geraet: 'Florian'. +[USER]: Ab jetzt immer auf der Kueche abspielen. +[ASSISTANT]: Gemerkt, ich nutze ab jetzt die Kueche. +-> [{"key": "spotify_hauptgeraet_name", "value": "Bevorzugtes Spotify-Wiedergabegeraet ist ab jetzt 'Kueche'. Nicht mehr 'Florian' als Standard verwenden.", "category": "spotify"}] +""" + +_CONSOLIDATION_SYSTEM_PROMPT = """\ +You are a memory maintenance assistant for the voice assistant Javis. +You receive the complete list of stored memory entries of one memory store. +Each line shows: key [category] (recalled Nx, updated YYYY-MM-DD): value + +Propose a SHORT list of cleanup operations: + * merge: entries that describe the same fact or rule under different keys. + Keep the most recent / most specific value; combine details when useful. + * delete: entries contradicted by a newer entry, or clearly one-off + observations with no future value. + +Be conservative — when in doubt, KEEP the entry unchanged. +Never merge unrelated facts. Never delete explicit user instructions unless +directly contradicted by a newer entry. Frequently recalled entries +(high "recalled Nx") are valuable — prefer keeping their content. +At most {MAX_OPS} operations. + +Respond with a JSON array ONLY (no prose, no markdown fences): +[ + {"op": "delete", "key": "key_to_delete"}, + {"op": "merge", "keys": ["old_key_1", "old_key_2"], "key": "new_snake_case_key", "value": "merged actionable rule", "category": "smart_home"} +] +If nothing needs cleanup: [] +""" + + +class MemoryMiddleware(AgentMiddleware): + """Auto-recall and auto-store memories around the agentic loop. + + **Per-speaker scoping** + - Identified speakers get a personal memory file: ``memory_{speaker}.json`` + (safe snake_case of the lowercased name). Facts whose category is + ``personal``, ``preferences``, or ``tasks`` are stored there. + - Household-level facts (``smart_home``, ``spotify``, ``facts``, ``context``) + go to the shared ``memory.json``, which is also written by the memory MCP + server and is therefore visible to all speakers. + - Anonymous speakers (typed text, unknown voices): recall uses the shared + household store only; extraction still runs, but personal-category facts + are dropped because they cannot be attributed to anyone. + + On ``on_before_llm`` (iteration 0 only): + - Injects ``[CURRENT SPEAKER: {name}]`` into the system prompt so the LLM + knows throughout the entire turn who it is talking to. + - Merges relevant facts from the speaker's personal file and the shared + household file into an ``[AUTO-RECALLED MEMORIES — {name}]`` block. + + On ``on_finish``: defers a background LLM call that extracts new facts + from the completed turn and routes them to the correct file by category. + The deferred coroutine never blocks the current or next turn. + """ + + name = "memory" + + def __init__(self, llm: "OllamaClient | HermesClient", config: "Config") -> None: + self._llm = llm + self._config = config + self._mem_cfg = config.memory + self._memory_file: Path = ( + Path(config.paths.base_dir) / "memory.json" + ) + # Serializes concurrent background write batches so two simultaneous + # extraction jobs can't overwrite each other's results. + self._write_lock: asyncio.Lock = asyncio.Lock() + # mtime-based in-process cache keyed by file Path. + # Each value is (file_mtime_float, parsed_dict). + # Invalidated (key deleted) immediately after a write to that file. + self._mem_cache: dict[Path, tuple[float, dict]] = {} + # (store_path, key) pairs recalled per turn — flushed to disk as + # recall_count/last_recalled_at stats in on_finish. + self._recalled_by_turn: dict[str, list[tuple[Path, str]]] = {} + # Turn counter driving the periodic consolidation pass. + self._turn_count: int = 0 + # Log which model will be used for extraction so config is verifiable. + mem_model = self._mem_cfg.model or config.llm.model + log.info( + f"[memory] extraction model: {mem_model} " + f"(temp={self._mem_cfg.temperature}, num_ctx={self._mem_cfg.num_ctx})" + ) + + # ── piped hook ──────────────────────────────────────────────────────────── + + async def on_before_llm( + self, + ctx: TurnContext, + messages: list[dict], + iteration: int, + ) -> Optional[list[dict]]: + """Inject speaker identity + recalled memories into the system message.""" + if iteration > 0: + # Tool-call re-iterations: skip — the block is already in messages[0]. + return None + + speaker = ctx.speaker_name # None when anonymous / speaker-ID disabled + + extra = "" + if speaker: + # Always inject the speaker identity so the LLM knows who it addresses. + extra += f"\n\n[CURRENT SPEAKER: {speaker}]" + + user_text = _last_user_text(messages) + if user_text: + # Follow-up turns ("mach das nochmal") carry few significant + # words — widen the query with the previous user message so + # recall keeps the preceding topic's facts available. + query = user_text + significant = [ + w for w in query.lower().split() + if len(w) >= _RECALL_MIN_WORD_LEN and w not in _RECALL_STOPWORDS + ] + if len(significant) < 3: + prev = _previous_user_text(messages) + if prev: + query = f"{prev}\n{query}" + + # Anonymous speakers still get recall from the shared household + # store (the system prompt promises automatic memory for all input + # paths, including typed text). + matches, is_fallback = await asyncio.to_thread( + self._search_memories, query, speaker + ) + if matches: + block = "\n".join( + f"• {key}: {entry.get('value', '')}" + for _, key, entry in matches + ) + if is_fallback: + extra += "\n\n[STANDING RULES — apply when relevant]\n" + block + else: + label = speaker or "household" + extra += ( + f"\n\n[AUTO-RECALLED MEMORIES — {label}]\n" + block + ) + # Track surfaced entries; recall stats are flushed after the turn. + self._recalled_by_turn[ctx.turn_id] = [ + (path, key) for path, key, _ in matches + ] + log.info( + f"[memory] recalled {len(matches)} fact(s) " + f"({'standing rules' if is_fallback else 'keyword match'}) " + f"→ injected into context (turn {ctx.turn_id}, speaker={speaker!r})" + ) + + if not extra: + return None + modified = list(messages) + sys_msg = modified[0] + modified[0] = {**sys_msg, "content": sys_msg["content"] + extra} + return modified + + # ── sequential hook ─────────────────────────────────────────────────────── + + # Tools whose successful (ok=True) results carry no new information for the + # extraction LLM and would only produce duplicate "use zone command" entries. + # Failed calls from these tools are still logged so the LLM can learn from them. + _SKIP_OK_TOOLS: frozenset[str] = frozenset( + {"homey__control_zone", "homey__control_all"} + ) + + async def on_after_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + result: str, + ok: bool, + latency: float, + ) -> None: + """Accumulate a compact tool-call summary for the extraction LLM. + + Filtering rules (prevent duplicate smart-home pattern extraction): + - ``javis__*`` tools (quick commands, timers) are bookkeeping only — skip always. + - Successful calls to ``homey__control_zone`` / ``homey__control_all`` produce + no new learnable fact (the zone workflow is already established) — skip if ok. + Failed calls from these tools ARE logged so the LLM can learn from errors. + - Everything else (homey__find_devices, spotify, MCP tools, etc.) is logged + so the LLM has context for extraction. + """ + # Always skip internal Javis bookkeeping tools. + if tool_name.startswith("javis__"): + return + # Skip successful routine zone/control calls — they carry no new information. + if ok and tool_name in self._SKIP_OK_TOOLS: + return + + args_str = json.dumps(args, ensure_ascii=False)[:120] + result_preview = result[:200].replace("\n", " ") + status = "OK" if ok else "ERR" + ctx.tool_log.append( + f"{tool_name}({args_str}) → {status}: {result_preview}" + ) + + async def on_finish( + self, + ctx: TurnContext, + user_text: str, + reply: str, + ) -> None: + """Defer background fact extraction if the turn had substantial content. + + Anonymous speakers are included — household-level facts (smart_home, + spotify, facts) are still worth storing; personal-category facts are + dropped at write time because they cannot be attributed. + """ + # Extract if tools were used (recurring commands form behavioural patterns + # worth remembering) OR if the user message is long enough to likely + # contain preferences, corrections, or explicit memory instructions. + # Skip only truly trivial turns: short text with zero tool calls. + _EXTRACT_MIN_USER_LEN = 25 + explicit_memory_request = bool(re.search( + r"\b(remember|memorize|save this|note this|merke|merk dir|speicher)\b", + user_text, + flags=re.IGNORECASE, + )) + should_extract = ( + bool(ctx.tool_log) + or len(user_text) >= _EXTRACT_MIN_USER_LEN + or explicit_memory_request + ) + if should_extract: + log.info( + f"[memory] scheduling background extraction " + f"(turn {ctx.turn_id}, user={len(user_text)}ch reply={len(reply)}ch" + f", tools={len(ctx.tool_log)}, speaker={ctx.speaker_name!r})" + ) + ctx.defer( + self._extract_and_store( + ctx.turn_id, user_text, reply, + list(ctx.tool_log), ctx.speaker_name, + ) + ) + else: + log.debug( + f"[memory] extraction skipped — trivial turn " + f"(turn {ctx.turn_id}, user={len(user_text)}ch < {_EXTRACT_MIN_USER_LEN}ch" + f", no tool calls)" + ) + + # Flush recall-usage stats recorded in on_before_llm. + recalled = self._recalled_by_turn.pop(ctx.turn_id, None) + if recalled: + ctx.defer(self._bump_recall_stats(recalled)) + + # Periodic background maintenance: merge duplicates, drop stale facts. + self._turn_count += 1 + if self._turn_count % _CONSOLIDATE_EVERY_TURNS == 0: + log.info( + f"[memory] scheduling consolidation pass " + f"(turn count {self._turn_count})" + ) + ctx.defer(self._consolidate(ctx.speaker_name)) + + async def on_abort(self, ctx: TurnContext, reason: str) -> None: + self._recalled_by_turn.pop(ctx.turn_id, None) + + async def on_error(self, ctx: TurnContext, exc: Exception) -> None: + self._recalled_by_turn.pop(ctx.turn_id, None) + + # ── internal helpers ────────────────────────────────────────────────────── + + def _speaker_file(self, speaker_name: str) -> Path: + """Return the personal memory file path for *speaker_name*. + + Uses a safe lowercase snake_case filename so arbitrary names don't + produce path-traversal issues or OS-illegal filenames. + e.g. "Florian" → ``~/.javis/memory_florian.json`` + """ + safe = re.sub(r"[^a-z0-9]", "_", speaker_name.lower()).strip("_") or "speaker" + return self._memory_file.parent / f"memory_{safe}.json" + + def _load_memory_data(self, path: Optional[Path] = None) -> dict: + """Return parsed memory JSON at *path*, using a per-path mtime cache. + + Skips re-parsing if the file has not changed since the last read. + Must only be called from a thread (via ``asyncio.to_thread``). + """ + target = path if path is not None else self._memory_file + try: + mtime = target.stat().st_mtime + except FileNotFoundError: + return {} + cached = self._mem_cache.get(target) + if cached is not None and cached[0] == mtime: + return cached[1] + try: + data = json.loads(target.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + data = {} + self._mem_cache[target] = (mtime, data) + return data + + def _search_memories( + self, + query: str, + speaker_name: Optional[str] = None, + fallback: bool = True, + ) -> tuple[list[tuple[Path, str, dict]], bool]: + """Return scored memory entries matching *query*. + + Returns ``(matches, is_fallback)`` where each match is + ``(store_path, key, entry)``, best matches first. + + Searches two stores — personal facts win ties: + 1. Speaker's personal file (``memory_{speaker}.json``) if identified. + 2. Shared household file (``memory.json``) — device layout, Spotify, etc. + + Matching is stopword-filtered keyword search, ranked by the number of + distinct matched words (key matches weigh double). When nothing + matches and *fallback* is True, standing behavioural rules + (``preferences``/``smart_home``) are returned instead, newest first. + """ + words = { + w for w in (t.strip(".,!?;:\"'").lower() for t in query.split()) + if len(w) >= _RECALL_MIN_WORD_LEN and w not in _RECALL_STOPWORDS + } + + # Load all stores up front — personal store first so its facts win. + stores: list[tuple[Path, str]] = [] + if speaker_name: + stores.append((self._speaker_file(speaker_name), f"personal:{speaker_name}")) + stores.append((self._memory_file, "shared")) + + loaded: list[tuple[Path, dict]] = [] + for path, label in stores: + try: + data = self._load_memory_data(path) + except Exception as exc: + log.warning(f"[memory] _search_memories ({label}): load failed: {exc}") + data = {} + loaded.append((path, data.get("entries", {}) or {})) + + # Score every entry: 2 pts per distinct word matching the key, + # 1 pt per distinct word matching only the value. + scored: list[tuple[int, int, Path, str, dict]] = [] + seen: set[str] = set() + for store_rank, (path, entries) in enumerate(loaded): + for key, entry in entries.items(): + if key in seen: + continue + seen.add(key) + key_l = key.lower() + val_l = str(entry.get("value", "")).lower() + score = sum( + 2 if w in key_l else 1 + for w in words + if w in key_l or w in val_l + ) + if score > 0: + scored.append((score, -store_rank, path, key, entry)) + if scored: + scored.sort(key=lambda t: (t[0], t[1]), reverse=True) + return ( + [(p, k, e) for _, _, p, k, e in scored[:_RECALL_MAX_RESULTS]], + False, + ) + + if not fallback: + return [], False + + # Fallback: no keyword hits — inject only standing behavioural rules + # (always applicable) instead of arbitrary recent facts. + rules: list[tuple[str, Path, str, dict]] = [] + rule_keys: set[str] = set() + for path, entries in loaded: + for key, entry in entries.items(): + if key in rule_keys: + continue + if entry.get("category") in _FALLBACK_CATEGORIES: + rule_keys.add(key) + rules.append((str(entry.get("updated_at", "")), path, key, entry)) + rules.sort(key=lambda t: t[0], reverse=True) + return [(p, k, e) for _, p, k, e in rules[:_RECALL_MAX_RESULTS]], True + + async def _extract_and_store( + self, + turn_id: str, + user_text: str, + reply: str, + tool_log: "list[str] | None" = None, + speaker_name: Optional[str] = None, + ) -> None: + """Background coroutine: run a lightweight LLM call to extract facts. + + Scheduled via ``ctx.defer()`` so it never blocks the current turn. + Existing stored memories and the turn's tool-call log are fed to the + LLM so it has full context for extraction. + Key events are logged at INFO; errors at WARNING. + """ + log.info(f"[memory] extracting facts from turn {turn_id}…") + snippet = reply[:_EXTRACT_REPLY_CAP] + user_snippet = user_text[:_EXTRACT_USER_TEXT_CAP] + + # Pass existing memories to the LLM so it avoids re-extracting known facts. + existing_summary = await asyncio.to_thread( + self._load_existing_summary, speaker_name + ) + existing_block = ( + f"[EXISTING MEMORIES]\n{existing_summary}\n\n" + if existing_summary else "" + ) + + # Identify the speaker in the extraction prompt so the LLM can + # personalise keys and values. + speaker_block = ( + f"[SPEAKER: {speaker_name}]\n\n" if speaker_name else "" + ) + + # Pass a compact tool-call log so the LLM can learn from tool outcomes. + tool_block = "" + if tool_log: + tool_lines = "\n".join(f" {line}" for line in tool_log[:5]) + tool_block = f"[TOOL CALLS THIS TURN]\n{tool_lines}\n\n" + + extraction_messages = [ + { + "role": "system", + "content": _EXTRACTION_SYSTEM_PROMPT.replace( + "{MAX_FACTS}", str(self._config.memory.max_facts) + ), + }, + { + "role": "user", + "content": ( + f"{speaker_block}" + f"{existing_block}" + f"{tool_block}" + f"[USER]: {user_snippet}\n" + f"[ASSISTANT]: {snippet}" + ), + }, + ] + try: + response = await self._llm.chat( + extraction_messages, + tools=None, + stream=False, + model=self._mem_cfg.model or None, + temperature=self._mem_cfg.temperature, + num_ctx=self._mem_cfg.num_ctx, + timeout=float(self._mem_cfg.timeout_seconds) + if self._mem_cfg.timeout_seconds is not None + else None, + ) + content = (response.content or "").strip() + + # Reasoning models (deepseek-r1, qwen3-thinking, etc.) emit a + # ... block before their JSON answer. Strip it so + # the JSON parser always sees clean output. + content = re.sub(r".*?", "", content, flags=re.DOTALL).strip() + + # Tolerate the model wrapping the array in prose: find the JSON array. + start = content.find("[") + end = content.rfind("]") + 1 + if start < 0 or end <= start: + log.info(f"[memory] no new facts found in turn {turn_id}") + return + + facts: list[dict] = json.loads(content[start:end]) + if not isinstance(facts, list): + log.info(f"[memory] no new facts found in turn {turn_id}") + return + + # Build a normalized key→(value, category) map from the LLM output. + to_store: dict[str, tuple[str, str]] = {} + for fact in facts[:self._config.memory.max_facts]: + if not isinstance(fact, dict): + log.warning(f"[memory] extraction: non-dict item in LLM response, skipping: {fact!r}") + continue + raw_key = str(fact.get("key", "")) + value = str(fact.get("value", "")).strip() + category = (str(fact.get("category", "")).strip() or "auto") + key = self._normalize_key(raw_key) + + # Validate the fact before storing to prevent random/hallucinated memories + if not key: + log.debug(f"[memory] extraction: empty normalized key, skipping fact: {raw_key!r}") + continue + + if not value: + log.debug(f"[memory] extraction: empty value for key: {key!r}") + continue + + # Apply content validation to filter out random/non-memorable facts + if self._validate_fact(key, value, category): + to_store[key] = (value, category) + else: + log.debug(f"[memory] extraction: rejected fact as non-memorable - key={key!r}") + + if not to_store: + log.info(f"[memory] no new facts found in turn {turn_id}") + return + + # Acquire the write lock so concurrent extraction jobs (one per turn) + # can't overwrite each other's results via a TOCTOU race. + async with self._write_lock: + stored = await asyncio.to_thread( + self._write_entries_batch, to_store, speaker_name + ) + + if stored: + log.info(f"[memory] stored {stored} new fact(s) from turn {turn_id}") + else: + log.info(f"[memory] no new facts found in turn {turn_id}") + + except Exception as exc: # noqa: BLE001 + exc_name = type(exc).__name__.lower() + if "timeout" in exc_name or "timeout" in str(exc).lower(): + mem_model = self._mem_cfg.model or self._config.llm.model + log.warning( + f"[memory] extraction timed out after {self._mem_cfg.timeout_seconds}s " + f"(turn {turn_id}, model={mem_model}). " + "Increase memory.timeout_seconds or use a faster model." + ) + else: + log.warning(f"[memory] extraction failed (turn {turn_id}): {exc}") + + @staticmethod + def _normalize_key(raw: str) -> str: + """Normalize an LLM-generated fact key to canonical ``snake_case``. + + Converts ``"User Name"``, ``"user-name"``, ``"user name"`` → ``"user_name"``. + Caps length at 64 characters to avoid pathological inputs. + """ + key = raw.strip().lower() + key = re.sub(r"[^a-z0-9]+", "_", key) + key = re.sub(r"_+", "_", key).strip("_") + # Return empty string for keys that normalise to nothing so the caller + # can filter them out. Do NOT fall back to "fact" — all bad keys would + # collide on the same dict entry and silently overwrite each other. + return key[:64] + + @staticmethod + def _validate_fact(key: str, value: str, category: str) -> bool: + """Validate that a fact is worth storing as memory. + + Rejects facts that appear to be random observations, LLM hallucinations, + or non-memorable content based on content analysis. + + Args: + key: The normalized snake_case key for the fact + value: The factual value to store + category: The category of the fact (personal, preferences, smart_home, etc.) + + Returns: + True if the fact should be stored, False if it should be rejected. + """ + # Reject keys that look like random observations or LLM artifacts + suspicious_key_patterns = [ + r"^the\s+user", # "the_user_said_something" - generic observation + r"\s+is\s+\w+$", # Keys ending with simple descriptions: "what_is_music" + r"^\d{4,}", # Keys starting with years or numbers: "2024_reminder" + r"(observation|fact|info)_\d+", # Auto-generated numbered keys + r"^remember_this", # Instruction-like keys from LLM confusion + ] + + for pattern in suspicious_key_patterns: + if re.search(pattern, key.lower()): + log.debug(f"[memory] rejected fact - suspicious key pattern: {key!r}") + return False + + # Identity facts ("Heißt Florian", a birthday) are legitimately short; + # only behavioural categories need rule-length values. + min_len = 4 if category in ("personal", "facts", "tasks") else 15 + if len(value) < min_len: + log.debug(f"[memory] rejected fact - value too short ({len(value)} chars): {key!r}") + return False + + # Reject generic phrases that indicate non-memorable content + generic_phrases = [ + "the user", # Generic LLM phrasing + "this is", # Descriptive, not actionable + "it was", # Passive observation + "i think", # Uncertainty marker + "maybe", # Uncertainty marker + "perhaps", # Uncertainty marker + ] + + value_lower = value.lower() + # Personal/identity facts are naturally phrased descriptively + # ("The user lives in Berlin") — only screen behavioural categories. + if category not in ("personal", "facts"): + for phrase in generic_phrases: + if phrase in value_lower and len(value) < 50: # Only flag short values with these phrases + log.debug(f"[memory] rejected fact - generic phrase '{phrase}' in value: {key!r}") + return False + + # For smart_home category, require actionable content + if category == "smart_home": + action_words = [ + "use", "verwende", "immer", "nicht", "kein", "schalte", "steuerung", + # device/zone naming facts are actionable layout knowledge + "heißt", "heisst", "name", "zone", "gerät", "geraet", "device", + ] + if not any(word in value_lower for word in action_words): + log.debug(f"[memory] rejected smart_home fact - no actionable content: {key!r}") + return False + + # For spotify category, require device/playlist references + if category == "spotify": + spotify_indicators = ["device", "gerät", "playlist", "artist", "song", "music"] + if not any(indicator in value_lower for indicator in spotify_indicators): + log.debug(f"[memory] rejected spotify fact - no device/playlist reference: {key!r}") + return False + + # Reject values that look like LLM uncertainty or confusion markers + uncertainty_markers = [ + r"i don't know", + r"I'm not sure", + r"ich weiß nicht", + r"vielleicht ist das", # "maybe it is..." - incomplete thoughts + r"das könnte", # "that could be..." - speculative + ] + + for marker in uncertainty_markers: + if re.search(marker, value_lower): + log.debug(f"[memory] rejected fact - uncertainty marker in value: {key!r}") + return False + + # Long-term memory must be timeless: reject values anchored to "now" + # (relative-time markers mean the fact is stale within hours/days). + # Tasks are exempt — an active task may legitimately reference a deadline. + if category != "tasks": + transient_markers = [ + r"\bheute\b", r"\bmorgen\b", r"\bübermorgen\b", r"\bgestern\b", + r"\bgerade\b", r"\baktuell\w*\b", r"\bmomentan\b", r"\bim moment\b", + r"\btoday\b", r"\btomorrow\b", r"\btonight\b", r"\byesterday\b", + r"\bcurrently\b", r"\bright now\b", r"\bthis (week|morning|evening)\b", + ] + for marker in transient_markers: + if re.search(marker, value_lower): + log.debug( + f"[memory] rejected fact - transient time marker in value: {key!r}" + ) + return False + + return True + + @staticmethod + def _values_similar(a: str, b: str, threshold: float = 0.75) -> bool: + """Word-overlap (Jaccard) similarity check for near-duplicate values.""" + wa = set(re.findall(r"\w{3,}", a.lower())) + wb = set(re.findall(r"\w{3,}", b.lower())) + if not wa or not wb: + return a.strip().lower() == b.strip().lower() + return len(wa & wb) / len(wa | wb) >= threshold + + def _load_existing_summary(self, speaker_name: Optional[str] = None) -> str: + """Return a brief human-readable list of stored facts for the extraction LLM. + + Merges personal facts (speaker file) + shared facts (household file), + sorted by most-recently-updated first, capped at + ``config.memory.max_existing_for_prompt`` entries. + """ + combined: dict[str, dict] = {} + for path in [ + *([ self._speaker_file(speaker_name) ] if speaker_name else []), + self._memory_file, + ]: + try: + data = self._load_memory_data(path) + for k, v in data.get("entries", {}).items(): + if k not in combined: + combined[k] = v + except Exception: + log.debug(f"[memory] failed to load entries from {path}") + if not combined: + return "" + sorted_entries = sorted( + combined.items(), + key=lambda kv: kv[1].get("updated_at", ""), + reverse=True, + ) + return "\n".join( + f"• {key}: {entry.get('value', '')}" + for key, entry in sorted_entries[:self._config.memory.max_existing_for_prompt] + ) + + def _write_entries_batch( + self, + entries: "dict[str, tuple[str, str]]", + speaker_name: Optional[str] = None, + ) -> int: + """Atomically merge *entries* into the correct memory file(s). + + Facts whose category is in ``_PERSONAL_CATEGORIES`` go to the + speaker's personal file; everything else goes to the shared household + file ``memory.json``. Both files use the same ``{entries: {}}`` schema + so the memory MCP server stays backward-compatible. + + Must be called while holding ``self._write_lock``. + Returns the total number of entries written, or 0 on failure. + """ + # Partition entries by destination file. + personal_path = self._speaker_file(speaker_name) if speaker_name else None + personal: dict[str, tuple[str, str]] = {} + shared: dict[str, tuple[str, str]] = {} + for key, (value, category) in entries.items(): + if category in _PERSONAL_CATEGORIES: + if speaker_name: + personal[key] = (value, category) + else: + # Anonymous speaker: personal facts cannot be attributed — + # drop them rather than pollute the shared household store. + log.info( + f"[memory] dropped personal fact {key!r} — anonymous speaker" + ) + else: + shared[key] = (value, category) + + total = 0 + for path, batch in [ + (personal_path, personal), + (self._memory_file, shared), + ]: + if not batch or path is None: + continue + total += self._write_to_file(path, batch) + + return total + + def _write_to_file( + self, + path: Path, + entries: "dict[str, tuple[str, str]]", + ) -> int: + """Atomically write *entries* into the JSON memory file at *path*. + + Preserves ``created_at`` for existing keys. Invalidates the mtime + cache for *path* after a successful write. + Returns the number of entries written, or 0 on failure. + """ + # Invalidate the per-path cache before writing. + self._mem_cache.pop(path, None) + try: + path.parent.mkdir(parents=True, exist_ok=True) + try: + data: dict = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + data = {"entries": {}} + + now = datetime.now(timezone.utc).isoformat() + written = 0 + for key, (value, category) in entries.items(): + existing = data["entries"].get(key) + # Skip near-duplicates: same key with essentially the same value, + # or the same content already stored under a different key. + if existing and self._values_similar(existing.get("value", ""), value): + log.info(f"[memory] dedup: {key!r} unchanged — skipping write") + continue + if not existing: + dup_key = next( + (k for k, e in data["entries"].items() + if self._values_similar(e.get("value", ""), value)), + None, + ) + if dup_key is not None: + log.info( + f"[memory] dedup: {key!r} duplicates existing " + f"{dup_key!r} — skipping write" + ) + continue + data["entries"][key] = { + "value": value, + "category": category, + "created_at": existing["created_at"] if existing else now, + "updated_at": now, + } + written += 1 + log.info(f"[memory] ✓ [{category}] {key!r} = {value!r} → {path.name}") + + if not written: + return 0 + + tmp = path.with_suffix(".json.part") + try: + tmp.write_text( + json.dumps(data, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + tmp.replace(path) + except Exception: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + log.debug(f"[memory] failed to atomically write {path.name}") + raise + + return written + + except Exception as exc: # noqa: BLE001 + log.warning(f"[memory] _write_to_file({path.name}) failed: {exc}") + return 0 + + def _atomic_write_json(self, path: Path, data: dict) -> bool: + """Write *data* to *path* atomically; invalidates the mtime cache.""" + self._mem_cache.pop(path, None) + tmp = path.with_suffix(".json.part") + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp.write_text( + json.dumps(data, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + tmp.replace(path) + return True + except Exception as exc: # noqa: BLE001 + log.warning(f"[memory] atomic write to {path.name} failed: {exc}") + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + return False + + # ── recall-usage tracking ──────────────────────────────────────────────── + + async def _bump_recall_stats(self, items: "list[tuple[Path, str]]") -> None: + """Persist recall_count/last_recalled_at for entries surfaced this turn.""" + by_path: dict[Path, list[str]] = {} + for path, key in items: + by_path.setdefault(path, []).append(key) + async with self._write_lock: + for path, keys in by_path.items(): + await asyncio.to_thread(self._bump_stats_file, path, keys) + + def _bump_stats_file(self, path: Path, keys: list[str]) -> None: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return + entries = data.get("entries", {}) or {} + now = datetime.now(timezone.utc).isoformat() + changed = False + for key in keys: + entry = entries.get(key) + if not isinstance(entry, dict): + continue + entry["recall_count"] = int(entry.get("recall_count", 0)) + 1 + entry["last_recalled_at"] = now + changed = True + if changed: + self._atomic_write_json(path, data) + + # ── on-demand memory tools (javis__memory_*) ───────────────────────────── + + async def search_memories_tool( + self, query: str, speaker_name: Optional[str] + ) -> str: + """On-demand memory search backing the ``javis__memory_search`` tool.""" + matches, _ = await asyncio.to_thread( + self._search_memories, query, speaker_name, False + ) + if not matches: + return "No stored memories match that query." + return "\n".join( + f"• {key} [{entry.get('category', '?')}]: {entry.get('value', '')}" + for _, key, entry in matches + ) + + async def forget_tool( + self, key_or_query: str, speaker_name: Optional[str] + ) -> str: + """Delete a stored memory — backs the ``javis__memory_forget`` tool. + + Deletes by exact (normalized) key first; otherwise by unique fuzzy + match on key/value. Multiple matches return the candidate keys so + the model can retry with the exact key. + """ + paths: list[Path] = [] + if speaker_name: + paths.append(self._speaker_file(speaker_name)) + paths.append(self._memory_file) + async with self._write_lock: + return await asyncio.to_thread(self._forget_sync, key_or_query, paths) + + def _forget_sync(self, key_or_query: str, paths: list[Path]) -> str: + norm = self._normalize_key(key_or_query) + needle = key_or_query.strip().lower() + loaded: list[tuple[Path, dict]] = [] + for path in paths: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + continue + loaded.append((path, data)) + + # Exact key match wins (personal store first). + for path, data in loaded: + entries = data.get("entries", {}) or {} + if norm and norm in entries: + removed = entries.pop(norm) + if self._atomic_write_json(path, data): + log.info(f"[memory] forgot {norm!r} ({path.name})") + return ( + f"Deleted memory '{norm}' " + f"(was: {removed.get('value', '')!r})." + ) + return "ERROR: failed to write memory file." + + # Fuzzy: substring match on key or value. + candidates: list[tuple[Path, dict, str]] = [] + if needle: + for path, data in loaded: + for key, entry in (data.get("entries", {}) or {}).items(): + if ( + needle in key.lower() + or needle in str(entry.get("value", "")).lower() + ): + candidates.append((path, data, key)) + if not candidates: + return f"No stored memory matches {key_or_query!r}." + if len(candidates) == 1: + path, data, key = candidates[0] + removed = data["entries"].pop(key) + if self._atomic_write_json(path, data): + log.info(f"[memory] forgot {key!r} ({path.name})") + return ( + f"Deleted memory '{key}' " + f"(was: {removed.get('value', '')!r})." + ) + return "ERROR: failed to write memory file." + keys = ", ".join(k for _, _, k in candidates[:10]) + return ( + f"Multiple memories match: {keys}. " + "Call javis__memory_forget again with the exact key." + ) + + # ── background consolidation ───────────────────────────────────────────── + + async def _consolidate(self, speaker_name: Optional[str]) -> None: + """Background maintenance: merge duplicate keys, drop stale facts.""" + paths = [self._memory_file] + if speaker_name: + paths.insert(0, self._speaker_file(speaker_name)) + for path in paths: + try: + await self._consolidate_file(path) + except Exception as exc: # noqa: BLE001 + log.warning( + f"[memory] consolidation failed for {path.name}: {exc}" + ) + + async def _consolidate_file(self, path: Path) -> None: + data = await asyncio.to_thread(self._load_memory_data, path) + entries = data.get("entries", {}) or {} + if len(entries) < _CONSOLIDATE_MIN_ENTRIES: + log.debug( + f"[memory] consolidation skipped for {path.name} " + f"({len(entries)} entries < {_CONSOLIDATE_MIN_ENTRIES})" + ) + return + + listing = "\n".join( + f"- {key} [{e.get('category', '?')}] " + f"(recalled {e.get('recall_count', 0)}x, " + f"updated {str(e.get('updated_at', ''))[:10]}): {e.get('value', '')}" + for key, e in entries.items() + ) + messages = [ + { + "role": "system", + "content": _CONSOLIDATION_SYSTEM_PROMPT.replace( + "{MAX_OPS}", str(_CONSOLIDATE_MAX_OPS) + ), + }, + {"role": "user", "content": listing}, + ] + response = await self._llm.chat( + messages, + tools=None, + stream=False, + model=self._mem_cfg.model or None, + temperature=self._mem_cfg.temperature, + num_ctx=self._mem_cfg.num_ctx, + timeout=float(self._mem_cfg.timeout_seconds) + if self._mem_cfg.timeout_seconds is not None + else None, + ) + content = re.sub( + r".*?", "", response.content or "", flags=re.DOTALL + ).strip() + start = content.find("[") + end = content.rfind("]") + 1 + if start < 0 or end <= start: + log.info(f"[memory] consolidation: no ops for {path.name}") + return + try: + ops = json.loads(content[start:end]) + except json.JSONDecodeError as exc: + log.warning(f"[memory] consolidation: bad JSON from LLM: {exc}") + return + if not isinstance(ops, list) or not ops: + log.info(f"[memory] consolidation: no ops for {path.name}") + return + + async with self._write_lock: + applied = await asyncio.to_thread( + self._apply_consolidation_ops, path, ops + ) + log.info( + f"[memory] consolidation applied {applied} op(s) to {path.name}" + ) + + def _apply_consolidation_ops(self, path: Path, ops: list) -> int: + """Apply validated delete/merge ops; returns the number applied.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return 0 + entries = data.get("entries", {}) or {} + now = datetime.now(timezone.utc).isoformat() + applied = 0 + for op in ops[:_CONSOLIDATE_MAX_OPS]: + if not isinstance(op, dict): + continue + kind = op.get("op") + if kind == "delete": + key = str(op.get("key", "")) + if key in entries: + log.info(f"[memory] consolidate: delete {key!r} ({path.name})") + del entries[key] + applied += 1 + elif kind == "merge": + keys = [ + k for k in (op.get("keys") or []) + if isinstance(k, str) and k in entries + ] + new_key = self._normalize_key(str(op.get("key", ""))) + value = str(op.get("value", "")).strip() + category = str(op.get("category", "")).strip() or "facts" + if len(keys) < 2 or not new_key or not value: + continue + merged_recalls = sum( + int(entries[k].get("recall_count", 0)) for k in keys + ) + created = min(entries[k].get("created_at", now) for k in keys) + for k in keys: + del entries[k] + entries[new_key] = { + "value": value, + "category": category, + "created_at": created, + "updated_at": now, + "recall_count": merged_recalls, + } + log.info( + f"[memory] consolidate: merged {keys} → {new_key!r} " + f"({path.name})" + ) + applied += 1 + if applied: + data["entries"] = entries + if not self._atomic_write_json(path, data): + return 0 + return applied + + +# ── Tool-cache middleware ───────────────────────────────────────────────────── + +class ToolCacheMiddleware(AgentMiddleware): + """LRU cache for tool call results. + + Analogous to TanStack's built-in ``toolCacheMiddleware``. + + Cache hits return a ``SkipToolDecision`` so the tool is not re-executed. + Only successful tool calls are cached — errors are never stored. + + *Not* added to the default middleware stack; opt in via:: + + orchestrator._middleware.append( + ToolCacheMiddleware(ttl_s=120, tool_names=["homey__list_devices"]) + ) + """ + + name = "tool_cache" + + def __init__( + self, + max_size: int = 50, + ttl_s: float = 300.0, + tool_names: Optional[list[str]] = None, + ) -> None: + self._max_size = max_size + self._ttl_s = ttl_s + # None means cache all tools; a set means cache only the listed tools. + self._tool_names: Optional[set[str]] = ( + set(tool_names) if tool_names is not None else None + ) + # Ordered dict used as an LRU cache: oldest entry at the front. + self._cache: dict[str, tuple[str, float]] = {} + + # ── first-win hook ──────────────────────────────────────────────────────── + + async def on_before_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + ) -> Optional[BeforeToolDecision]: + if self._tool_names is not None and tool_name not in self._tool_names: + return None + + cache_key = json.dumps([tool_name, args], sort_keys=True) + entry = self._cache.get(cache_key) + if entry is not None: + result, ts = entry + if time.time() - ts < self._ttl_s: + # LRU-promote: move to the back (most-recently-used). + self._cache.pop(cache_key) + self._cache[cache_key] = (result, ts) + log.debug( + f"[tool_cache] HIT {tool_name} " + f"(age {time.time()-ts:.0f}s, ttl {self._ttl_s:.0f}s)" + ) + return SkipToolDecision(result=result) + else: + # Expired — evict. + self._cache.pop(cache_key, None) + return None + + # ── sequential hook ─────────────────────────────────────────────────────── + + async def on_after_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + result: str, + ok: bool, + latency: float, + ) -> None: + if not ok: + return + if self._tool_names is not None and tool_name not in self._tool_names: + return + + cache_key = json.dumps([tool_name, args], sort_keys=True) + + # Evict oldest entry if at capacity. + if cache_key not in self._cache and len(self._cache) >= self._max_size: + oldest = next(iter(self._cache)) + self._cache.pop(oldest) + + self._cache[cache_key] = (result, time.time()) + log.debug(f"[tool_cache] STORE {tool_name}") + + +# ── Homey guard middleware ──────────────────────────────────────────────────── + + +def _is_placeholder_uuid(uuid_str: str) -> bool: + """Return True if *uuid_str* looks like an invented/placeholder UUID. + + Catches patterns like ``00000000-1111-2222-3333-444444444444`` where every + segment of the UUID consists of a single repeated character, as well as the + all-zeros nil UUID. + """ + s = uuid_str.strip().lower() + if s == "00000000-0000-0000-0000-000000000000": + return True + parts = s.split("-") + if len(parts) == 5: + return all(len(set(p)) == 1 for p in parts) + return False + + +class HomeyGuardMiddleware(AgentMiddleware): + """Block Homey tool calls that carry an obviously fake/placeholder ID. + + When a model emits a placeholder UUID such as + ``00000000-1111-2222-3333-444444444444`` for a device, flow, or logic + variable, this middleware short-circuits the call before it reaches the + network and returns a clear error directing the model to call the + matching discovery tool (``find_devices`` / ``find_flows`` / + ``list_variables``) first. + """ + + name = "homey_guard" + + # Argument keys that carry a Homey-assigned UUID needing discovery first, + # mapped to the tool the model should call to obtain the real value. + _ID_ARGS = { + "device_id": "homey__find_devices", + "flow_id": "homey__find_flows", + "variable_id": "homey__list_variables", + } + + async def on_before_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + ) -> Optional[BeforeToolDecision]: + if not tool_name.startswith("homey__"): + return None + for arg_name, discovery_tool in self._ID_ARGS.items(): + value = str(args.get(arg_name, "")) + if value and _is_placeholder_uuid(value): + log.warning( + f"[homey_guard] blocked {tool_name} — placeholder UUID " + f"'{value}' in '{arg_name}'; model must call {discovery_tool} first" + ) + return SkipToolDecision( + result=( + f"ERROR: '{value}' is a placeholder UUID, not a real Homey " + f"{arg_name}. You MUST call {discovery_tool} with the name " + "first to obtain the real UUID, then retry this tool." + ) + ) + return None + + +class PrivilegeMiddleware(AgentMiddleware): + """Enforce per-speaker tool access based on privileges attached to TurnContext. + + The orchestrator sets ``turn_ctx.privileges`` after speaker identification. + When a tool is about to be called this middleware checks whether it is + permitted under the current speaker's ``SpeakerPrivileges``. If not, it + returns a :class:`SkipToolDecision` so the LLM receives an access-denied + result instead of the tool executing. + + The ``can_respond`` flag is enforced *before* the LLM is invoked — checked + directly by the orchestrator after it sets ``turn_ctx.privileges``. + """ + + name = "privilege" + + async def on_before_tool( + self, + ctx: TurnContext, + tool_name: str, + args: dict, + ) -> Optional[BeforeToolDecision]: + priv = ctx.privileges + if priv is not None and not priv.allows_tool(tool_name): + speaker_label = ctx.speaker_name or "unknown speaker" + log.info( + f"[privilege] tool '{tool_name}' blocked for {speaker_label}" + ) + return SkipToolDecision( + result=( + f"[Access denied: the tool '{tool_name}' is not permitted " + f"for {speaker_label}.]" + ) + ) + return None + + +# ── Pipeline runners ────────────────────────────────────────────────────────── + +async def run_before_llm( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + messages: list[dict], + iteration: int, +) -> list[dict]: + """Pipe messages through all middleware ``on_before_llm`` hooks. + + Each middleware receives the (possibly modified) output of the previous one. + Returning ``None`` from a hook means "pass through unchanged". + + Analogous to TanStack's piped ``onConfig`` composition rule. + """ + current = messages + for mw in middlewares: + try: + modified = await mw.on_before_llm(ctx, current, iteration) + if modified is not None: + current = modified + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_before_llm raised: {exc}") + return current + + +async def run_before_tool( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + tool_name: str, + args: dict, +) -> Optional[BeforeToolDecision]: + """Call each middleware's ``on_before_tool`` until one returns a decision. + + Analogous to TanStack's first-win ``onBeforeToolCall`` composition rule. + """ + for mw in middlewares: + try: + decision = await mw.on_before_tool(ctx, tool_name, args) + if decision is not None: + return decision + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_before_tool raised: {exc}") + return None + + +async def run_after_tool( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + tool_name: str, + args: dict, + result: str, + ok: bool, + latency: float, +) -> None: + """Call all middleware ``on_after_tool`` hooks in order. + + Analogous to TanStack's sequential ``onAfterToolCall`` composition rule. + """ + for mw in middlewares: + try: + await mw.on_after_tool(ctx, tool_name, args, result, ok, latency) + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_after_tool raised: {exc}") + + +async def run_finish( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + user_text: str, + reply: str, +) -> None: + """Call all middleware ``on_finish`` hooks in order. + + Analogous to TanStack's sequential terminal ``onFinish`` composition rule. + """ + for mw in middlewares: + try: + await mw.on_finish(ctx, user_text, reply) + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_finish raised: {exc}") + + +async def run_start( + middlewares: list[AgentMiddleware], + ctx: TurnContext, +) -> None: + """Call all middleware ``on_start`` hooks in order. + + Analogous to TanStack's sequential ``onStart``. + """ + for mw in middlewares: + try: + await mw.on_start(ctx) + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_start raised: {exc}") + + +async def run_abort( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + reason: str, +) -> None: + """Call all middleware ``on_abort`` hooks in order. + + Analogous to TanStack's sequential ``onAbort``. + """ + for mw in middlewares: + try: + await mw.on_abort(ctx, reason) + except Exception as exc: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_abort raised: {exc}") + + +async def run_error( + middlewares: list[AgentMiddleware], + ctx: TurnContext, + exc: Exception, +) -> None: + """Call all middleware ``on_error`` hooks in order. + + Analogous to TanStack's sequential ``onError``. + """ + for mw in middlewares: + try: + await mw.on_error(ctx, exc) + except Exception as e2: # noqa: BLE001 + log.warning(f"[middleware] {mw.name}.on_error raised: {e2}") + + +# ── Private helpers ─────────────────────────────────────────────────────────── + +def _last_user_text(messages: list[dict]) -> str: + """Return the content of the most recent user message, or empty string.""" + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content", "") + return content if isinstance(content, str) else "" + return "" + + +def _previous_user_text(messages: list[dict]) -> str: + """Return the content of the second-most-recent user message, or ''.""" + found = 0 + for msg in reversed(messages): + if msg.get("role") == "user": + found += 1 + if found == 2: + content = msg.get("content", "") + return content if isinstance(content, str) else "" + return "" diff --git a/src/javis/agent/orchestrator.py b/src/javis/agent/orchestrator.py new file mode 100644 index 0000000..de6062c --- /dev/null +++ b/src/javis/agent/orchestrator.py @@ -0,0 +1,3300 @@ +"""Agent orchestrator — wires STT ↔ LLM ↔ MCP ↔ TTS via a state machine.""" +from __future__ import annotations + +import asyncio +import json +import re +import time +from dataclasses import dataclass +from typing import Any, Optional + +import numpy as np + +from ..audio_io.capture import record_utterance +from ..audio_io.stream import MicStream, play_pcm +from ..audio_io.vad import make_vad +from ..audio_io.wake_word import WakeWordDetector +from ..core.config import Config +from ..core.events import EventBus +from ..core.logger import get_logger +from ..core.state import AgentState +from ..llm.conversation import Conversation +from ..llm.hermes_client import HermesClient +from ..llm.ollama_client import ChatResponse, OllamaClient, ToolCall +from ..mcp_client.host import MCPHost +from ..stt.transcriber import Transcriber +from ..tts.factory import make_backend +from ..voices.manager import VoiceManager +from .quick_engine import QuickEngine +from ..core.timer import TimerService +from .middleware import ( + AgentMiddleware, + HomeyGuardMiddleware, + MemoryMiddleware, + PrivilegeMiddleware, + TurnContext, + run_before_llm, + run_before_tool, + run_after_tool, + run_finish, + run_start, + run_abort, + run_error, + AbortTurnDecision, + SkipToolDecision, + TransformArgsDecision, + drain_background_tasks, +) +from .prompts import load_system_prompt +from ..stt.voice_profiler import NullProfiler, SpeakerPrivileges, VoiceProfiler + +log = get_logger("agent") + +SENTENCE_BOUNDARY = re.compile(r"(?<=[\.\!\?…])\s+|(?<=[\.\!\?…])$") + +# Detects whether the last sentence of a reply is a question. +_FOLLOWUP_RE = re.compile(r'\?\s*$') + + +def _reply_expects_followup(reply: str) -> bool: + """Return True if *reply* ends with a question, implying the user should answer.""" + return bool(_FOLLOWUP_RE.search(reply.strip())) + + +def _normalize_for_speech(text: str) -> str: + """Convert LLM markdown output to plain speech-friendly prose. + + Strips all markup, ensures list items end as individual sentences, folds + line breaks into spaces, and removes bare URLs so that the downstream + sentence splitter produces natural, correctly-chunked TTS input. + """ + # 1. Fenced code blocks — skip entirely (unreadable) + text = re.sub(r'```[\s\S]*?```', '', text) + # 2. Bare URLs (https://…) — stop before brackets/parens to avoid + # eating the closing paren of a markdown link, e.g. [text](https://…) + text = re.sub(r'https?://[^\s\[\](){}<>"]+', '', text) + # 3. Horizontal rules (---, ***, ___) + text = re.sub(r'^[ \t]*[-*_]{3,}[ \t]*$', '', text, flags=re.MULTILINE) + # 4. Markdown links [text](url) → text + text = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', text) + # 5. Bold / italic markers (keep inner text) + text = re.sub(r'\*{1,3}([^*\n]*)\*{1,3}', r'\1', text) + text = re.sub(r'_{1,2}([^_\n]*)_{1,2}', r'\1', text) + # 6. Heading markers (## Title → Title) + text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE) + # 7. Inline code — keep content, strip backticks + text = re.sub(r'`([^`]+)`', r'\1', text) + # 8. Blockquotes + text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE) + # 9. List items: strip marker and ensure terminal sentence punctuation + # so the SENTENCE_BOUNDARY splitter can break on each item. + def _list_item(m: re.Match) -> str: + content = m.group(1).rstrip() + if content and content[-1] not in '.!?…:': + content += '.' + return content + text = re.sub(r'^[ \t]*(?:\d+\.|[-*+])\s+(.+)', _list_item, text, flags=re.MULTILINE) + # 10. Fold non-empty lines: if a line doesn't end with sentence punctuation + # add a period so the splitter creates a natural pause after each item. + def _fold_line(m: re.Match) -> str: + line = m.group(1) + if line and line[-1] not in '.!?…:,': + return line + '. ' + return line + ' ' + text = re.sub(r'([^\n]+)\n', _fold_line, text) + # 11. Collapse any remaining newlines (blank lines between blocks) + text = re.sub(r'\n+', ' ', text) + # 12. Em-dash / en-dash → brief prosodic pause + text = text.replace('—', ', ').replace('–', ', ') + # 13. Clean up artefacts from the above transforms + text = re.sub(r'[ \t]{2,}', ' ', text) # multiple spaces → one + text = re.sub(r'([.!?])\s*\.', r'\1', text) # stray double period + text = re.sub(r':\s*[.]', ':', text) # ":." → ":" + text = re.sub(r',\s*,', ',', text) # double comma + return text.strip() + + +_SPEECH_CHUNK_MAX_CHARS: int = 220 + + +def _group_speech_chunks(text: str) -> list[str]: + """Split *text* into TTS-friendly chunks for natural prosody. + + Strategy: normalise markdown → split on sentence boundaries → greedily + merge short fragments into chunks of at most ``_SPEECH_CHUNK_MAX_CHARS`` + characters. Feeding TTS longer, multi-sentence inputs produces a + continuous pitch arc instead of the flat reset you get from one sentence + per call. + """ + normalized = _normalize_for_speech(text) + frags = [s.strip() for s in SENTENCE_BOUNDARY.split(normalized) if s.strip()] + if not frags: + return [normalized] if normalized else [] + chunks: list[str] = [] + buf = "" + for frag in frags: + candidate = buf + " " + frag if buf else frag + if buf and len(candidate) > _SPEECH_CHUNK_MAX_CHARS: + chunks.append(buf) + buf = frag + else: + buf = candidate + if buf: + chunks.append(buf) + return chunks + + +# Maximum characters stored per tool result in conversation history. +# Raw MCP results can be very large (file listings, track lists, etc.); +# storing them verbatim inflates the context window permanently across the +# session and eventually causes CUDA OOM or prompt-length failures. +_MAX_TOOL_RESULT_CHARS = 2000 + + +def _truncate_tool_result(result: str, cap: int = _MAX_TOOL_RESULT_CHARS) -> str: + """Cap a tool result for history, truncating JSON at item level when possible. + + Blind character truncation can cut JSON mid-object, leaving the model to + reason over broken data. For JSON arrays/objects the payload is trimmed + at element boundaries with an explicit omission marker instead. + """ + if len(result) <= cap: + return result + try: + data = json.loads(result) + except (ValueError, TypeError): + data = None + if isinstance(data, list): + kept: list = [] + size = 2 + for item in data: + s = json.dumps(item, ensure_ascii=False) + if size + len(s) + 60 > cap: + break + kept.append(item) + size += len(s) + 2 + omitted = len(data) - len(kept) + if kept and omitted > 0: + kept.append(f"[…{omitted} of {len(data)} items omitted]") + return json.dumps(kept, ensure_ascii=False) + elif isinstance(data, dict): + kept_d: dict = {} + size = 2 + for k, v in data.items(): + s = json.dumps({k: v}, ensure_ascii=False) + if size + len(s) + 60 > cap: + break + kept_d[k] = v + size += len(s) + omitted = len(data) - len(kept_d) + if kept_d and omitted > 0: + kept_d["_truncated"] = f"…{omitted} of {len(data)} keys omitted" + return json.dumps(kept_d, ensure_ascii=False) + return result[:cap] + f"\n…[{len(result) - cap} chars truncated]" + + +# ── text-mode tool call detection ──────────────────────────────────────────── +# Some models (notably qwen2.5) occasionally emit tool calls as plain text +# rather than using the structured function-calling interface. Common forms: +# -server__tool {"arg": "val"} (qwen2.5 dash-prefix) +# server__tool({"arg": "val"}) (JSON args in parens) +# {"name":"...","arguments":{}} +# server__tool(key="val", key2=5) (kwargs, possibly inside a code fence) +# ```javascript\nserver__tool(key="val")\n``` +_TEXT_TC_RE = re.compile( + r'-?\s*([\w]+__[\w]+)\s*\(?\s*(\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\})\s*\)?', + re.DOTALL, +) +_XML_TC_RE = re.compile(r'\s*(\{.*?\})\s*', re.DOTALL) +# Kwargs form: first non-whitespace arg token starts with \w (not '{'). +_KWARGS_TC_RE = re.compile( + r'([\w]+__[\w]+)\s*\(\s*(\w[^()]*?)\s*\)', + re.DOTALL, +) +_CODE_FENCE_RE = re.compile(r'```[^\n]*\n(.*?)```', re.DOTALL) + + +def _parse_kwargs(s: str) -> dict | None: + """Convert ``key="val", key2=5`` into a plain dict. + + Supports double/single-quoted strings, JSON literals (int, float, + bool, null) and Python True/False/None. Returns ``None`` when a + positional (non-keyword) argument is encountered. + """ + result: dict = {} + parts: list[str] = [] + buf: list[str] = [] + in_quote: str | None = None + for ch in s: + if in_quote: + buf.append(ch) + if ch == in_quote: + in_quote = None + elif ch in ('"', "'"): + in_quote = ch + buf.append(ch) + elif ch == ',': + parts.append(''.join(buf).strip()) + buf = [] + else: + buf.append(ch) + if buf: + parts.append(''.join(buf).strip()) + + for part in parts: + part = part.strip() + if not part: + continue + if '=' not in part: + return None # positional arg — give up + key, _, raw = part.partition('=') + key = key.strip() + raw = raw.strip() + if not key: + return None + try: + result[key] = json.loads(raw) + continue + except json.JSONDecodeError: + pass + if raw.startswith("'") and raw.endswith("'") and len(raw) >= 2: + result[key] = raw[1:-1] + continue + if raw == 'True': + result[key] = True + elif raw == 'False': + result[key] = False + elif raw == 'None': + result[key] = None + else: + result[key] = raw + + return result if result else None + + +def _parse_text_tool_calls(content: str) -> tuple[list[ToolCall], str]: + """Extract tool calls the model emitted as plain text instead of structured calls. + + Returns ``(extracted_calls, cleaned_content)`` where *cleaned_content* has + the matched spans removed so the remaining prose can be surfaced normally. + """ + calls: list[ToolCall] = [] + spans: list[tuple[int, int]] = [] + + for m in _TEXT_TC_RE.finditer(content): + try: + args = json.loads(m.group(2)) + except (json.JSONDecodeError, ValueError): + continue + calls.append(ToolCall(name=m.group(1), arguments=args)) + spans.append((m.start(), m.end())) + + for m in _XML_TC_RE.finditer(content): + try: + obj = json.loads(m.group(1)) + except (json.JSONDecodeError, ValueError): + continue + name = obj.get("name", "") + args = obj.get("arguments", {}) + if name and "__" in name: + calls.append(ToolCall(name=name, arguments=args)) + spans.append((m.start(), m.end())) + + # ── kwargs / code-fence form ────────────────────────────────────────── + # e.g. ```javascript\nduckduckgo__ddg_news(query="trump news", max_results=5)\n``` + # or inline: tool__name(key="val", key2=5) + + def _span_free(start: int, end: int) -> bool: + return not any(s < end and e > start for s, e in spans) + + # Try fenced code blocks first — remove the whole fence to avoid orphaned ```. + for fence_m in _CODE_FENCE_RE.finditer(content): + if not _span_free(fence_m.start(), fence_m.end()): + continue + for kw_m in _KWARGS_TC_RE.finditer(fence_m.group(1)): + kw = _parse_kwargs(kw_m.group(2)) + if kw is not None: + calls.append(ToolCall(name=kw_m.group(1), arguments=kw)) + spans.append((fence_m.start(), fence_m.end())) + break # one tool call per fence block + + # Also match kwargs directly in the body (outside already-captured regions). + for m in _KWARGS_TC_RE.finditer(content): + if not _span_free(m.start(), m.end()): + continue + kw = _parse_kwargs(m.group(2)) + if kw is not None: + calls.append(ToolCall(name=m.group(1), arguments=kw)) + spans.append((m.start(), m.end())) + + if not calls: + return [], content + + # Remove matched spans in reverse order to preserve character offsets. + remainder = content + for start, end in sorted(spans, reverse=True): + remainder = remainder[:start] + remainder[end:] + remainder = re.sub(r'\n{3,}', '\n\n', remainder).strip() + return calls, remainder + + +@dataclass +class RemoteTurnResult: + """Outcome of a turn submitted by a javis-slave device.""" + user_text: str = "" + reply_text: str = "" + audio: Optional[np.ndarray] = None # float32 mono reply speech + sample_rate: Optional[int] = None + + +class Orchestrator: + """Long-lived voice agent. Owns all subsystems.""" + + def __init__(self, config: Config, bus: Optional[EventBus] = None): + self.config = config + self.bus = bus or EventBus() + self.state: AgentState = AgentState.IDLE + + self.mic: MicStream | None = MicStream(config) if config.stt.enabled else None + self.vad = make_vad(threshold=config.wake_word.vad_threshold) + # Voice input requires STT; if STT is disabled, also disable wake word. + self.stt = Transcriber(config) if config.stt.enabled else None + self.wake = None + if config.wake_word.enabled and self.stt is not None: + try: + self.wake = WakeWordDetector( + config.wake_word.model, config.wake_word.threshold, + search_dirs=[config.paths.models_dir / "wakeword"]) + except Exception as exc: # noqa: BLE001 + log.warning( + f"wake-word detector unavailable ({exc}) — " + "falling back to push-to-talk only" + ) + self.wake = None + self.tts = make_backend(config) + self.voices = VoiceManager(config) + self.llm = OllamaClient(config) + self.memory_llm = ( + HermesClient(config) if config.memory.provider == "hermes" else self.llm + ) + self.mcp = MCPHost(config) + self.timer_service = TimerService( + self._on_timer_fire, + persist_path=config.paths.base_dir / "timers.json", + ) + + self.conv = Conversation( + system_prompt=load_system_prompt( + config.agent.system_prompt_file, + inline=config.agent.system_prompt_inline, + preset=config.agent.preset, + ), + max_turns=config.agent.history_max_turns, + ) + + self._cancel_playback = asyncio.Event() + self._stop = asyncio.Event() + # Prevent push-to-talk and wake-loop from running a turn concurrently. + self._turn_lock = asyncio.Lock() + # Set after a reply that ends with a question; run() will skip + # wake-word detection for the next turn so the user can answer + # without saying the wake phrase again. + self._followup_pending: bool = False + # Set after a reply that ends with '?'; _handle_turn will triple the + # listening timeout for the next recording so users have more time to respond. + self._last_response_is_question: bool = False + # Barge-in: set by the wake-word listener while Javis is speaking when + # the user says the wake phrase again. run() consumes it to jump + # straight into the next listening turn (skipping _wait_for_wake). + self._barge_in_requested: bool = False + self._barge_in_task: Optional[asyncio.Task] = None + + # Composable hook-based middleware pipeline — inspired by TanStack AI. + # Each element is an AgentMiddleware instance; order matters for piped + # and first-win hooks. Append ToolCacheMiddleware or custom middleware + # here to extend the pipeline without touching the loop logic. + self._middleware: list[AgentMiddleware] = [ + PrivilegeMiddleware(), + HomeyGuardMiddleware(), + MemoryMiddleware(llm=self.memory_llm, config=self.config), + ] + # Direct handle for the javis__memory_* built-in tools. + self._memory_mw: MemoryMiddleware = self._middleware[-1] # type: ignore[assignment] + # Flag controlled by _llm_loop to decide whether a turn should start + # a fresh conversation or continue as an assistant follow-up reply. + self._turn_turn_start: bool = True + + self.profiler: VoiceProfiler | NullProfiler = ( + VoiceProfiler(config) if config.speaker_id.enabled else NullProfiler() + ) + self._enroll_clip_buffer: list[np.ndarray] = [] + self._enroll_ask_pending: bool = False + self._bg_quick_turn_count: int = 0 + self._tool_cache: dict[str, tuple[str, float]] = {} + + self.quick = QuickEngine(config.paths.base_dir / "quick_commands.json") + self.quick.load() + + def register_middleware(self, middleware: AgentMiddleware) -> None: + """Insert *middleware* at the front of the pipeline (after Privilege). + + The pipeline is executed in order for sequential hooks and piped through + ``on_before_llm`` / ``on_before_tool``, so inserting early gives the new + middleware first dibs on every turn or tool call. Use this instead of + directly mutating ``_middleware`` when extending Javis with plugins or + additional cross-cutting concerns. + """ + # PrivilegeMiddleware must stay at index 0 (checked in _llm_loop). + self._middleware.insert(1, middleware) + + @property + def middleware(self) -> list["AgentMiddleware"]: + """Read-only snapshot of the active middleware pipeline (for inspection).""" + return list(self._middleware) + + @property + def turn_start(self) -> bool: + """Whether the current turn should start a fresh conversation. + + Controlled by :meth:`_llm_loop` to decide whether the assistant message + belongs to a new turn (tool calls + final reply) or is a follow-up reply + after a previous tool call round-trip. Exposed on ``orchestrator`` so + downstream middleware (e.g. MemoryMiddleware) can inspect it when building + prompts. + """ + return self._turn_turn_start + + @turn_start.setter + def turn_start(self, value: bool) -> None: + self._turn_turn_start = value + + # ---------------- lifecycle ---------------- + + async def set_state(self, s: AgentState) -> None: + self.state = s + await self.bus.publish("state", s) + log.info(f"state -> {s.value}") + + async def start(self) -> None: + log.info("starting subsystems …") + await self.bus.publish("task_start", {"id": "boot", "name": "Loading models & MCP servers"}) + + async def _phase(tid: str, name: str, coro): + await self.bus.publish("task_start", {"id": tid, "name": name}) + ok = True + err = "" + try: + return await coro + except Exception as e: # noqa: BLE001 + ok = False + err = str(e) + log.exception(f"{name} failed") + raise + finally: + await self.bus.publish("task_done", + {"id": tid, "ok": ok, "msg": err or "done"}) + + if self.stt is not None and self.mic is not None: + try: + await self.mic.start() + except Exception as exc: + # No local microphone available — common on Remote Desktop sessions, + # or when the master runs headless. Degrade gracefully to text-only + # / remote-only mode rather than crashing. NOTE: self.stt is kept + # alive — it doesn't depend on local mic hardware and is still + # needed to transcribe audio uploaded by javis-slave devices. + log.warning( + f"Microphone unavailable ({exc}) — disabling local voice input " + "(text input and slave-device turns still work)." + ) + self.mic = None + self.wake = None + await self.bus.publish("error", { + "where": "mic", + "message": ( + f"Microphone unavailable: {exc}. " + "Local voice input disabled (text input and slave devices still work)." + ), + }) + else: + log.info("STT disabled — mic not started (text-only mode)") + load_tasks = [ + _phase("boot.mcp", "Connecting MCP servers", self.mcp.start()), + ] + if self.stt is not None: + load_tasks.insert(0, _phase("boot.stt", "Loading speech-to-text", + asyncio.to_thread(self.stt.load))) + else: + log.info("STT disabled — voice input unavailable (text input still works)") + if self.tts is not None: + load_tasks.insert(1, _phase("boot.tts", "Loading text-to-speech", + asyncio.to_thread(self.tts.load))) + else: + log.info("TTS disabled — running text-only") + if self.config.speaker_id.enabled: + load_tasks.append(_phase("boot.profiler", "Loading voice profiler", + asyncio.to_thread(self.profiler.load))) + else: + log.info("Speaker ID disabled") + # Use return_exceptions=True so that a failure in one subsystem (e.g. STT + # model not found) does not cancel the others. Each _phase() already + # publishes a task_done{ok=False} event on exception, so the GUI updates + # correctly. Individual exceptions are logged here for traceability. + results = await asyncio.gather(*load_tasks, return_exceptions=True) + for r in results: + if isinstance(r, BaseException): + log.error(f"subsystem startup failed: {r}") + # Disable any backend whose load() raised so later synthesis/prewarm + # calls don't encounter RuntimeError("load() has not been called"). + if self.tts is not None and getattr(self.tts, "_voice", True) is None: + log.warning("TTS model failed to load — speech output disabled") + self.tts = None + await self.bus.publish("mcp_ready", self.mcp.status_snapshot()) + ok = await self.llm.health() + await self.bus.publish("llm_health", {"ok": ok, "url": self.config.llm.base_url, "model": self.config.llm.model}) + if not ok: + log.warning(f"Ollama at {self.config.llm.base_url} unreachable — chat will fail") + if isinstance(self.memory_llm, HermesClient): + try: + models = await self.memory_llm.list_models() + memory_ok = bool(models) + await self.bus.publish("memory_health", { + "ok": memory_ok, + "url": self.config.memory.base_url, + "models": models, + }) + if not memory_ok: + log.warning("Hermes is reachable but advertised no models") + except Exception as exc: # noqa: BLE001 + await self.bus.publish("memory_health", { + "ok": False, + "url": self.config.memory.base_url, + "error": str(exc), + }) + log.warning(f"Hermes memory provider is not ready: {exc}") + await self.set_state(AgentState.IDLE) + await self.bus.publish("task_done", {"id": "boot", "ok": True, "msg": "ready"}) + log.info("Javis ready.") + + # Reschedule any timers that were still pending when Javis last exited. + try: + await self.timer_service.restore() + except Exception: # noqa: BLE001 + log.exception("timer restore failed") + + # TTS pre-warm: synthesize a short dummy phrase in the background so + # that the TTS backend loads its model before the first real turn. The + # one-time penalty is absorbed during the idle wait for wake-word + # rather than added to the first spoken response. + if self.tts is not None: + asyncio.ensure_future(self._prewarm_tts()) + + async def stop(self) -> None: + self._stop.set() + await self.timer_service.aclose() + if self.mic is not None: + await self.mic.stop() + await self._stop_barge_in() + await self.mcp.stop() + await drain_background_tasks() + if self.memory_llm is not self.llm: + await self.memory_llm.aclose() + await self.llm.aclose() + + # ---------------- main loop ---------------- + + async def run(self) -> None: + try: + while not self._stop.is_set(): + try: + if self.stt is None or self.mic is None: + # No local voice input (STT disabled, or mic hardware + # unavailable) — only text submissions and slave-device + # turns drive this Orchestrator. + await asyncio.sleep(0.5) + continue + grace_ms = 1500 + if self.wake is not None: + if self._followup_pending: + # Last reply ended with a question — listen immediately + # without requiring the wake phrase again. + self._followup_pending = False + grace_ms = int(self.config.wake_word.followup_grace_s * 1000) + log.info("follow-up mode: skipping wake word") + await self.bus.publish("followup_listen", {}) + asyncio.ensure_future(self._play_wake_chime()) + elif self._barge_in_requested: + # User said the wake word while Javis was speaking — + # playback was cancelled; listen immediately. + self._barge_in_requested = False + if self.mic is not None: + # Discard the tail of the wake phrase so STT + # doesn't transcribe "hey jarvis" as the utterance. + self.mic.drain() + grace_ms = int(self.config.wake_word.followup_grace_s * 1000) + log.info("barge-in: skipping wake word") + await self.bus.publish("barge_in_listen", {}) + # Chime was already played by the barge-in listener. + else: + woke = await self._wait_for_wake() + if not woke: + # Stale timeout or mic disconnect — skip this turn + # and loop back to wait for a real wake event. + continue + if self._stop.is_set(): + break + async with self._turn_lock: + await self._handle_turn(initial_grace_ms=grace_ms) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 + log.exception(f"turn failed: {e}") + await self.bus.publish("error", {"where": "turn", "message": str(e)}) + await self.set_state(AgentState.ERROR) + # Voice users can't see the GUI error — say something. + await self._speak_notice(self.config.agent.error_phrase) + await asyncio.sleep(1.0) + # Recover to IDLE so the agent can accept new input after an error. + # Without this recovery, a single failure would leave Javis stuck + # in ERROR state forever. We transition to IDLE and let the main + # loop resume waiting for wake-word or text input. + await self.set_state(AgentState.IDLE) + await asyncio.sleep(0.5) + except asyncio.CancelledError: + log.info("orchestrator cancelled") + finally: + await self.stop() + + async def trigger_push_to_talk(self) -> None: + """Manually trigger a conversation turn (push-to-talk).""" + if self.stt is None or self.mic is None: + log.warning("PTT ignored — no local microphone available. Use the text input instead.") + await self.bus.publish("error", { + "where": "ptt", + "message": "Spracheingabe ist deaktiviert. Text-Eingabe verwenden.", + }) + return + if self._turn_lock.locked(): + log.info("turn already running — ignoring PTT") + await self.bus.publish("notice", { + "where": "ptt", + "message": "Javis ist noch beschäftigt — bitte warten.", + }) + return + async with self._turn_lock: + await self._handle_turn() + + async def submit_text(self, text: str, *, speak: bool = True) -> None: + """Run a conversation turn from typed text (bypasses STT). + + Reuses `_turn_lock` so it can't race with PTT / wake-word turns. If a + turn is already running the text is queued and sent right after it. + Reply is spoken via TTS unless `speak=False`. + """ + text = (text or "").strip() + if not text: + return + if self._turn_lock.locked(): + log.info("turn already running — queuing text submit") + await self.bus.publish("notice", { + "where": "text", + "message": "Javis ist noch beschäftigt — Nachricht wird danach gesendet.", + }) + async with self._turn_lock: + t0 = time.monotonic() + await self.set_state(AgentState.THINKING) + cue_task = self._start_thinking_cue() if speak else None + log.info(f"user (typed): {text}") + await self.bus.publish("user_text", text) + if await self._try_quick_response(text): + if cue_task is not None: + cue_task.cancel() + await self.set_state(AgentState.IDLE) + return + self.conv.add_user(text) + turn_ctx = TurnContext(turn_id=f"t{int(t0*1000)%100000}") + await run_start(self._middleware, turn_ctx) + tts_feed_sub: asyncio.Queue | None = None + speak_task_sub: asyncio.Task | None = None + try: + if speak: + tts_feed_sub = asyncio.Queue() + speak_task_sub = asyncio.create_task( + self._speak_stream(tts_feed_sub) + ) + reply = await self._llm_loop(turn_ctx, tts_feed=tts_feed_sub) + if reply: + await self.bus.publish("assistant_text", reply) + await run_finish(self._middleware, turn_ctx, text, reply) + await turn_ctx.flush() + if speak_task_sub is not None: + await speak_task_sub + if self.config.wake_word.followup_enabled: + ends_with_question = _reply_expects_followup(reply) + self._followup_pending = ( + ends_with_question and self.wake is not None + ) + # Also set flag to triple listening timeout on next recording + self._last_response_is_question = ends_with_question + asyncio.ensure_future(self._bg_quick_suggest(text)) + else: + if speak_task_sub is not None: + speak_task_sub.cancel() + try: + await speak_task_sub + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await run_abort(self._middleware, turn_ctx, "empty_reply") + log.info(f"text turn complete in {time.monotonic()-t0:.2f}s") + except Exception as exc: # noqa: BLE001 + await run_error(self._middleware, turn_ctx, exc) + log.exception(f"turn failed in {time.monotonic()-t0:.2f}s: {exc}") + await self.bus.publish("error", {"where": "llm", "message": str(exc)}) + if speak: + await self._speak_notice(self.config.agent.error_phrase) + finally: + if cue_task is not None: + cue_task.cancel() + if speak_task_sub is not None and not speak_task_sub.done(): + speak_task_sub.cancel() + try: + await speak_task_sub + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await self.set_state(AgentState.IDLE) + + async def submit_remote_turn( + self, + audio: np.ndarray, + sample_rate: int, + *, + room: Optional[str] = None, + device_name: str = "", + ) -> "RemoteTurnResult": + """Run a conversation turn from audio recorded on a javis-slave device. + + Mirrors :meth:`submit_text` but takes raw audio: transcribes it here on + the master, tags the user text with the slave's room, runs the normal + agentic LLM loop, and synthesizes the final reply once (no streaming + TTS — the audio is returned to the slave for local playback). + """ + t0 = time.monotonic() + if self.stt is None: + raise RuntimeError("STT is disabled on the master — cannot process remote audio") + + async with self._turn_lock: + await self.set_state(AgentState.THINKING) + + # ── parallel STT + speaker identification ──────────────────────── + # Same treatment as locally-captured audio: speaker ID runs + # alongside transcription so remote (slave) speakers get identified, + # privilege-checked and auto-enrolled/refined exactly like local mic + # input, instead of always being treated as an anonymous speaker. + if self.config.speaker_id.enabled: + tr, id_result = await asyncio.gather( + self.stt.transcribe(audio, sample_rate), + asyncio.to_thread(self.profiler.identify, audio, sample_rate), + ) + else: + tr = await self.stt.transcribe(audio, sample_rate) + id_result = None + text = (tr.text or "").strip() + if not text: + log.info(f"remote turn ({device_name!r}): empty transcription") + await self.set_state(AgentState.IDLE) + return RemoteTurnResult(user_text="", reply_text="") + + speaker_prefix = "" + _sp_name: Optional[str] = None + _privileges: SpeakerPrivileges = SpeakerPrivileges() # default: full access + if self.config.speaker_id.enabled: + if id_result: + sp_name, sp_conf = id_result + _sp_name = sp_name + speaker_prefix = f"[Speaker: {sp_name}] " + _privileges = self.profiler.get_privileges(sp_name) + await self.bus.publish( + "speaker_id", + {"name": sp_name, "confidence": sp_conf, "known": True}, + ) + log.info(f"[profiler] remote speaker={sp_name!r} conf={sp_conf:.3f}") + refine_min = ( + self.config.speaker_id.threshold + + self.config.speaker_id.refine_margin + ) + if self.config.speaker_id.auto_refine and sp_conf >= refine_min: + asyncio.create_task( + asyncio.to_thread( + self.profiler.refine, sp_name, audio, sample_rate, + ) + ) + else: + _privileges = self._unknown_privileges() + await self.bus.publish( + "speaker_id", {"name": None, "confidence": 0.0, "known": False} + ) + if self.config.speaker_id.auto_enroll: + await self._auto_enroll_speaker(audio) + + if not _privileges.can_respond: + log.info( + f"[privilege] remote speaker '{_sp_name or 'unknown'}' is not " + "permitted to receive a response — dropping turn" + ) + await self.bus.publish("privilege_denied", {"speaker": _sp_name}) + await self.set_state(AgentState.IDLE) + return RemoteTurnResult(user_text=text, reply_text="") + + room_prefix = f"[Room: {room}] " if room else "" + log.info(f"user (remote, {device_name!r}, room={room!r}): {text}") + await self.bus.publish("user_text", f"{room_prefix}{speaker_prefix}{text}") + self.conv.add_user(f"{room_prefix}{speaker_prefix}{text}") + + turn_ctx = TurnContext( + turn_id=f"r{int(t0*1000)%100000}", + privileges=_privileges, + speaker_name=_sp_name, + ) + await run_start(self._middleware, turn_ctx) + reply = "" + try: + reply = await self._llm_loop(turn_ctx, tts_feed=None) + if reply: + await self.bus.publish("assistant_text", reply) + await run_finish(self._middleware, turn_ctx, text, reply) + await turn_ctx.flush() + else: + await run_abort(self._middleware, turn_ctx, "empty_reply") + log.info(f"remote turn complete in {time.monotonic()-t0:.2f}s") + except Exception as exc: # noqa: BLE001 + await run_error(self._middleware, turn_ctx, exc) + log.exception(f"remote turn failed: {exc}") + raise + finally: + await self.set_state(AgentState.IDLE) + + # Synthesize the full reply once for slave-side playback (outside the + # turn lock — the master is free for new turns during synthesis). + reply_audio, reply_sr = await self.synthesize_speech(reply) + + return RemoteTurnResult( + user_text=text, + reply_text=reply, + audio=reply_audio, + sample_rate=reply_sr, + ) + + async def synthesize_speech(self, text: str) -> tuple[Optional[np.ndarray], Optional[int]]: + """Render *text* to a single float32 buffer for remote playback. + + Returns ``(None, None)`` when TTS is disabled, the voice has no + reference clips, or synthesis fails — callers fall back to text only. + """ + if not text or self.tts is None: + return None, None + try: + speech = _normalize_for_speech(text) + if not speech: + return None, None + profile = self.voices.get_or_create_default(self.config.tts.default_voice) + if not profile.reference_clips: + return None, None + parts: list[np.ndarray] = [] + sr = None + for chunk in _group_speech_chunks(speech): + result = await self.tts.synthesize(chunk, profile) + parts.append(np.asarray(result.audio, dtype=np.float32)) + sr = result.sample_rate + if parts and sr: + return np.concatenate(parts), sr + except Exception as exc: # noqa: BLE001 + log.warning(f"remote TTS failed (returning text only): {exc}") + return None, None + + # If the mic produces no audio for this long, restart the wake loop rather + # than blocking the coroutine forever (handles hot-unplug / driver freeze). + _WAKE_STALE_S: float = 60.0 + + async def _wait_for_wake(self) -> bool: + """Block until a wake-word is detected. Returns True on detection, + False when the listen loop exited without a detection (stale timeout, + ``_stop`` set, or mic disconnected).""" + assert self.wake is not None + await self.set_state(AgentState.IDLE) + self.wake.reset() + if self.mic is not None: + self.mic.drain() # discard audio buffered during STT/LLM — start fresh + log.info(f"listening for wake word: '{self.config.wake_word.model}'") + last_pub = 0.0 + async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S): + score = self.wake.feed(chunk) + now = time.monotonic() + if now - last_pub > 0.1: # ~10 Hz UI updates + last_pub = now + rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) if len(chunk) else 0.0 + await self.bus.publish("mic_level", {"rms": rms, "score": float(score)}) + if self.wake.triggered(score): + log.info(f"wake! score={score:.2f}") + await self.bus.publish("wake", score) + # Play a short chime so the user knows Javis is listening. + asyncio.ensure_future(self._play_wake_chime()) + return True + if self._stop.is_set(): + return False + # Generator exited — either stale timeout or mic sentinel. + return False + + async def _barge_in_listener(self) -> None: + """Watch for the wake word while Javis is speaking (barge-in). + + Lets the user interrupt TTS playback by saying the wake phrase again. + Runs concurrently with the TTS consumer, so it must be the *only* + other mic consumer at that time: ``_turn_lock`` is held across the + whole turn, so ``record_utterance`` cannot run simultaneously. On + trigger it cancels playback, resets the detector, and sets + ``_barge_in_requested`` so ``run()`` jumps straight into the next + listening turn. + """ + if self.wake is None or self.mic is None: + return + self.wake.reset() + self.mic.drain() # discard audio buffered during THINKING — start fresh + log.info("listening for barge-in while speaking") + try: + async for chunk in self.mic.chunks(stale_timeout=self._WAKE_STALE_S): + if self._stop.is_set(): + break + score = self.wake.feed(chunk) + if self.wake.triggered(score): + log.info(f"barge-in! score={score:.2f}") + await self.bus.publish("barge_in", score) + # Chime so the user knows Javis heard them. + asyncio.ensure_future(self._play_wake_chime()) + self.wake.reset() + self._barge_in_requested = True + self.interrupt() # -> _cancel_playback set + await self.set_state(AgentState.WAKE) + return + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + log.debug(f"barge-in listener exited (non-critical): {exc}") + + def _start_barge_in(self) -> None: + """Start the barge-in listener (no-op if disabled or no mic/wake).""" + if self._barge_in_task is not None: + return + if not self.config.wake_word.barge_in_enabled: + return + if self.wake is None or self.mic is None: + return + self._barge_in_task = asyncio.ensure_future(self._barge_in_listener()) + + async def _stop_barge_in(self) -> None: + """Cancel and await the barge-in listener, if one is running.""" + task = self._barge_in_task + if task is None: + return + self._barge_in_task = None + if not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + async def _play_wake_chime(self) -> None: + """Play a short two-tone chime to signal that the wake word was heard.""" + try: + sr = self.config.audio.sample_rate + out_device = self.config.audio.output_device + # Generate a quick ascending two-tone: 880 Hz → 1320 Hz, 60 ms each. + t1 = np.linspace(0, 0.06, int(sr * 0.06), endpoint=False, dtype=np.float32) + t2 = np.linspace(0, 0.06, int(sr * 0.06), endpoint=False, dtype=np.float32) + tone1 = (np.sin(2 * np.pi * 880 * t1) * 0.18).astype(np.float32) + tone2 = (np.sin(2 * np.pi * 1320 * t2) * 0.18).astype(np.float32) + chime = np.concatenate([tone1, tone2]) + await play_pcm(chime, sr, output_device=out_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"wake chime failed (non-critical): {exc}") + + async def _play_timer_chime(self) -> None: + """Play a three-tone ascending alarm chime to signal a timer has fired. + + Distinct from the wake chime (two tones, 880→1320 Hz) — uses three + louder tones at 880→1047→1319 Hz (A5, C6, E6 — a major chord arpeggio) + so the user instantly recognises it as a timer alert rather than a + wake event. + """ + try: + sr = self.config.audio.sample_rate + out_device = self.config.audio.output_device + dur = 0.12 # seconds per tone + fade = 0.008 # fade-out per tone to avoid clicks + freqs = [880.0, 1047.0, 1319.0] + tones = [] + for f in freqs: + n = int(sr * dur) + t = np.linspace(0, dur, n, endpoint=False, dtype=np.float32) + wave = np.sin(2 * np.pi * f * t) * 0.30 + # Short linear fade-out at the end of each tone. + fade_n = min(int(sr * fade), n) + wave[-fade_n:] *= np.linspace(1.0, 0.0, fade_n, dtype=np.float32) + tones.append(wave.astype(np.float32)) + chime = np.concatenate(tones) + await play_pcm(chime, sr, output_device=out_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"timer chime failed (non-critical): {exc}") + + async def _play_confirmation_chime(self) -> None: + """Play a short descending chime to confirm a quick command executed. + + Uses the opposite pitch direction to the wake chime (1320 → 880 Hz) + so the user can distinguish "Javis woke up" from "quick command ran". + """ + try: + sr = self.config.audio.sample_rate + out_device = self.config.audio.output_device + # Descending two-tone: 1320 Hz → 880 Hz, 50 ms each. + t1 = np.linspace(0, 0.05, int(sr * 0.05), endpoint=False, dtype=np.float32) + t2 = np.linspace(0, 0.05, int(sr * 0.05), endpoint=False, dtype=np.float32) + tone1 = (np.sin(2 * np.pi * 1320 * t1) * 0.15).astype(np.float32) + tone2 = (np.sin(2 * np.pi * 880 * t2) * 0.15).astype(np.float32) + chime = np.concatenate([tone1, tone2]) + await play_pcm(chime, sr, output_device=out_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"confirmation chime failed (non-critical): {exc}") + + async def _play_soft_tick(self) -> None: + """Very quiet single tone: "still working" cue during long thinking gaps.""" + try: + sr = self.config.audio.sample_rate + t = np.linspace(0, 0.045, int(sr * 0.045), endpoint=False, dtype=np.float32) + tone = np.sin(2 * np.pi * 660 * t) * 0.06 + fade_n = max(1, int(sr * 0.008)) + tone[-fade_n:] *= np.linspace(1.0, 0.0, fade_n, dtype=np.float32) + await play_pcm(tone.astype(np.float32), sr, + output_device=self.config.audio.output_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"thinking tick failed (non-critical): {exc}") + + async def _play_attention_chime(self) -> None: + """Two identical mid tones — "your input is needed on screen".""" + try: + sr = self.config.audio.sample_rate + t = np.linspace(0, 0.08, int(sr * 0.08), endpoint=False, dtype=np.float32) + tone = (np.sin(2 * np.pi * 988 * t) * 0.2).astype(np.float32) + gap = np.zeros(int(sr * 0.06), dtype=np.float32) + await play_pcm(np.concatenate([tone, gap, tone]), sr, + output_device=self.config.audio.output_device) + except Exception as exc: # noqa: BLE001 + log.debug(f"attention chime failed (non-critical): {exc}") + + def _start_thinking_cue(self) -> "asyncio.Task[None] | None": + """Schedule soft ticks while the agent stays in THINKING past the configured delay. + + Returns the task (cancel it when the turn ends) or None when disabled. + """ + delay = self.config.audio.thinking_cue_delay_s + if delay <= 0 or self.tts is None: + return None + + async def _loop() -> None: + await asyncio.sleep(delay) + while self.state == AgentState.THINKING: + await self._play_soft_tick() + await asyncio.sleep(4.0) + + return asyncio.create_task(_loop()) + + async def _speak_notice(self, text: str) -> None: + """Speak a short system phrase, then restore the previous agent state. + + Used for error/approval feedback outside the normal reply pipeline. + Never raises. + """ + if not text or self.tts is None: + return + prev = self.state + try: + await self._speak(text) + except Exception as exc: # noqa: BLE001 + log.debug(f"speak_notice failed (non-critical): {exc}") + finally: + if prev in (AgentState.THINKING, AgentState.ERROR) and self.state != prev: + await self.set_state(prev) + + async def notify_approval_pending(self, *, speak: bool) -> None: + """Audible cue that a tool call is waiting for on-screen approval.""" + await self._play_attention_chime() + if speak: + await self._speak_notice(self.config.agent.approval_phrase) + + _YES_WORDS = frozenset({ + "ja", "jawohl", "okay", "ok", "klar", "gerne", "mach", "erlauben", "erlaubt", + "genehmigt", "bitte", "yes", "yeah", "yep", "sure", "allow", "go", "do", "approve", + }) + _NO_WORDS = frozenset({ + "nein", "nee", "nö", "nicht", "stopp", "stop", "abbrechen", "abbruch", "verweigern", + "lass", "no", "nope", "don't", "dont", "cancel", "deny", "never", + }) + + async def approval_by_voice(self, timeout_s: float, *, speak: bool) -> Optional[bool]: + """Ask aloud and listen for a yes/no answer while the approval dialog is open. + + Returns True/False on a recognised answer, None when nothing usable + was heard before *timeout_s* (the caller keeps waiting on the dialog). + Safe to run during a turn: the turn lock is held, so no other mic + consumer is active (barge-in only runs while speaking). + """ + if not self.config.mcp.voice_approval or self.stt is None or self.mic is None: + return None + await self.notify_approval_pending(speak=speak) + deadline = time.monotonic() + max(3.0, timeout_s) + while not self._stop.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0.5: + return None + self.mic.drain() + try: + audio = await record_utterance( + self.config, self.mic, self.vad, + initial_grace_ms=int(min(6000.0, remaining * 1000)), + silence_ms_override=500, + ) + except Exception as exc: # noqa: BLE001 + log.debug(f"voice approval capture failed: {exc}") + return None + if audio is None or len(audio) < 1600: + continue + try: + tr = await self.stt.transcribe(audio, self.config.audio.sample_rate) + except Exception as exc: # noqa: BLE001 + log.debug(f"voice approval STT failed: {exc}") + return None + words = set(re.findall(r"[\w']+", tr.text.lower())) + log.info(f"voice approval heard: {tr.text!r}") + # "nein" wins over an accidental "ok" in the same utterance. + if words & self._NO_WORDS: + return False + if words & self._YES_WORDS: + return True + return None + + def _unknown_privileges(self) -> SpeakerPrivileges: + """Build a :class:`SpeakerPrivileges` from the unknown-speaker config fields.""" + cfg = self.config.speaker_id + denied = [p.strip() for p in cfg.unknown_denied_tools.split(",") if p.strip()] + raw_allowed = cfg.unknown_allowed_tools.strip() + allowed: Any = ( + "*" if raw_allowed == "*" + else [p.strip() for p in raw_allowed.split(",") if p.strip()] + ) + return SpeakerPrivileges( + can_respond=cfg.unknown_can_respond, + allowed_tools=allowed, + denied_tools=denied, + ) + + async def _prewarm_tts(self) -> None: + """Silently synthesize a short phrase to warm up TTS model buffers. + + Some backends pay a one-time cost on the first synthesis of each + session. Running this in the background immediately after startup + absorbs that cost while the user is still reaching for the wake phrase. + The resulting audio is discarded — it is never played or published. + """ + try: + t0 = time.monotonic() + profile = self.voices.get_or_create_default(self.config.tts.default_voice) + if not profile.reference_clips: + log.debug("[tts-prewarm] skipped — no reference clips for default voice") + return + await self.tts.synthesize("Bereit.", profile) + log.info(f"[tts-prewarm] complete in {time.monotonic() - t0:.2f}s") + except Exception as exc: # noqa: BLE001 + log.debug(f"[tts-prewarm] skipped (non-critical): {exc}") + + async def _handle_turn(self, initial_grace_ms: int = 1500) -> None: + if self.stt is None: + log.info("voice turn skipped — STT disabled") + return + await self.set_state(AgentState.LISTENING) + t0 = time.monotonic() + # If last response was a question, triple the listening timeout + silence_override = None + if self._last_response_is_question: + base_silence = self.config.wake_word.silence_ms + silence_override = base_silence * 3 + log.debug(f"last response was a question; extending silence timeout to {silence_override}ms (was {base_silence}ms)") + self._last_response_is_question = False # reset flag for next turn + audio = await record_utterance(self.config, self.mic, self.vad, + initial_grace_ms=initial_grace_ms, + silence_ms_override=silence_override) + if audio is None or len(audio) < 1600: + log.info("no utterance captured") + await self.bus.publish("stt_done", {"text": "", "audio_s": 0.0, "latency_s": 0.0, "empty": True}) + await self.bus.publish("stt_empty", {}) + # Reset to IDLE so the GUI doesn't stay stuck on "listening" + # when the turn ends without any speech. + await self.set_state(AgentState.IDLE) + return + + await self.set_state(AgentState.THINKING) + cue_task = self._start_thinking_cue() + audio_s = len(audio) / float(self.config.audio.sample_rate) + t_stt = time.monotonic() + + # ── parallel STT + speaker identification ──────────────────────────── + # Both tasks take the same raw audio buffer and are fully independent, + # so running them concurrently absorbs speaker-ID latency (150–500 ms) + # into the STT wait time at zero cost. + if self.config.speaker_id.enabled: + tr, id_result = await asyncio.gather( + self.stt.transcribe(audio, self.config.audio.sample_rate), + asyncio.to_thread(self.profiler.identify, audio, self.config.audio.sample_rate), + ) + else: + tr = await self.stt.transcribe(audio, self.config.audio.sample_rate) + id_result = None + stt_latency = time.monotonic() - t_stt + + speaker_prefix = "" + _sp_name: Optional[str] = None + _privileges: SpeakerPrivileges = SpeakerPrivileges() # default: full access + if self.config.speaker_id.enabled: + if id_result: + sp_name, sp_conf = id_result + _sp_name = sp_name + speaker_prefix = f"[Speaker: {sp_name}] " + _privileges = self.profiler.get_privileges(sp_name) + await self.bus.publish( + "speaker_id", + {"name": sp_name, "confidence": sp_conf, "known": True}, + ) + log.info(f"[profiler] speaker={sp_name!r} conf={sp_conf:.3f}") + # Auto-refine: blend this clip into the stored profile so that + # the centroid tracks the speaker's voice over time. Only use + # high-confidence matches (>= threshold + refine_margin) so that + # borderline clips don't erode profile quality. + refine_min = ( + self.config.speaker_id.threshold + + self.config.speaker_id.refine_margin + ) + if self.config.speaker_id.auto_refine and sp_conf >= refine_min: + log.debug( + f"[profiler] auto-refining '{sp_name}' " + f"(conf={sp_conf:.3f} >= {refine_min:.3f})" + ) + asyncio.create_task( + asyncio.to_thread( + self.profiler.refine, sp_name, audio, + self.config.audio.sample_rate, + ) + ) + elif self.config.speaker_id.auto_refine: + log.debug( + f"[profiler] skipping refine for '{sp_name}' " + f"(conf={sp_conf:.3f} < {refine_min:.3f})" + ) + else: + _privileges = self._unknown_privileges() + await self.bus.publish( + "speaker_id", {"name": None, "confidence": 0.0, "known": False} + ) + if self.config.speaker_id.auto_enroll: + await self._auto_enroll_speaker(audio) + + # ── privilege: can_respond check ────────────────────────────────────── + if not _privileges.can_respond: + log.info( + f"[privilege] speaker '{_sp_name or 'unknown'}' is not permitted " + "to receive a response — dropping turn" + ) + await self.bus.publish("privilege_denied", {"speaker": _sp_name}) + if cue_task is not None: + cue_task.cancel() + await self.set_state(AgentState.IDLE) + return + + await self.bus.publish("stt_done", { + "text": tr.text, "audio_s": audio_s, "latency_s": stt_latency, + "empty": not tr.text.strip(), + }) + if not tr.text.strip(): + log.info("empty transcription") + await self.bus.publish("stt_empty", {}) + if cue_task is not None: + cue_task.cancel() + await self.set_state(AgentState.IDLE) + return + log.info(f"user: {tr.text} (audio {audio_s:.1f}s, stt {stt_latency:.2f}s)") + await self.bus.publish("user_text", tr.text) + if await self._try_quick_response(tr.text): + if cue_task is not None: + cue_task.cancel() + await self.set_state(AgentState.IDLE) + return + self.conv.add_user(f"{speaker_prefix}{tr.text}") + + turn_ctx = TurnContext( + turn_id=f"v{int(t0*1000)%100000}", + privileges=_privileges, + speaker_name=_sp_name, + ) + await run_start(self._middleware, turn_ctx) + tts_feed: asyncio.Queue = asyncio.Queue() + speak_task = asyncio.create_task(self._speak_stream(tts_feed)) + try: + reply = await self._llm_loop(turn_ctx, tts_feed=tts_feed) + if reply: + await self.bus.publish("assistant_text", reply) + await run_finish(self._middleware, turn_ctx, tr.text, reply) + await turn_ctx.flush() + await speak_task + if self.config.wake_word.followup_enabled: + ends_with_question = _reply_expects_followup(reply) + self._followup_pending = ( + ends_with_question and self.wake is not None + ) + # Also set flag to triple listening timeout on next recording + self._last_response_is_question = ends_with_question + asyncio.ensure_future(self._bg_quick_suggest(tr.text)) + else: + speak_task.cancel() + try: + await speak_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await run_abort(self._middleware, turn_ctx, "empty_reply") + log.info(f"turn complete in {time.monotonic()-t0:.2f}s") + except Exception as exc: # noqa: BLE001 + await run_error(self._middleware, turn_ctx, exc) + raise + finally: + if cue_task is not None: + cue_task.cancel() + if not speak_task.done(): + speak_task.cancel() + try: + await speak_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + # ---------------- LLM with tools ---------------- + + # How often the background quick-command analyzer runs (every Nth LLM turn). + _BG_QUICK_EVERY_N: int = 3 + + # Keyword sets for routing tools to the relevant MCP server subset. + # Sending all 65 schemas to a 7-B model on a single GPU exhausts CUDA VRAM. + # Most turns only need one server; routing keeps the payload to ≤30 tools. + _HOMEY_KEYWORDS = frozenset({ + 'licht', 'lampe', 'hell', 'dunkel', 'dim', 'homey', 'gerät', 'geräte', + 'steckdose', 'heizung', 'thermostat', 'zimmer', 'zone', 'schalte', + 'schalten', 'ausschalten', 'einschalten', 'flow', 'szene', 'scene', + 'esszimmer', 'wohnzimmer', 'küche', 'schlafzimmer', 'bad', 'flur', + }) + _SPOTIFY_KEYWORDS = frozenset({ + 'musik', 'song', 'lied', 'spotify', 'playlist', 'album', 'artist', + 'abspielen', 'pause', 'weiter', 'skip', 'lauter', 'leiser', 'volume', + 'radio', 'spiel', + }) + _FILES_KEYWORDS = frozenset({ + 'datei', 'ordner', 'file', 'folder', 'code', 'skript', 'verzeichnis', + }) + _SEARCH_KEYWORDS = frozenset({ + # German — search / news / research intent + 'news', 'nachrichten', 'nachricht', 'tagesschau', 'bericht', 'berichte', + 'artikel', 'meldung', 'aktuell', 'neueste', 'neuesten', 'neuigkeiten', + 'suche', 'such', 'such nach', 'recherche', 'information', 'informationen', + 'internet', 'web', 'google', 'was ist passiert', 'was ist los', + # English — search / research intent + 'search', 'find', 'look up', 'lookup', 'research', 'news', 'article', + 'latest', 'recent', 'what happened', 'what is', 'how to', + }) + _SLAVE_KEYWORDS = frozenset({ + # German — system / device control + 'lautstärke', 'volumen', 'volume', 'ton', 'audio', 'sound', 'mute', 'stumm', + 'helligkeit', 'brightness', 'bildschirm', 'display', 'hell', 'dunkel', 'dim', + 'schlaf', 'sleep', 'herunterfahren', 'shutdown', 'ausschalten', 'neustart', + 'restart', 'neustarten', 'ruhezustand', 'hibernation', 'standby', 'rechner', + 'computer', 'device', 'slave', 'system', 'prozessor', 'cpu', 'monitor', + # English — system / device control + 'volume', 'louder', 'quieter', 'mute', 'unmute', 'brightness', 'screen', + 'sleep', 'shutdown', 'restart', 'reboot', 'turn off', 'wake', 'standby', + }) + _EMBY_KEYWORDS = frozenset({ + # German — video / streaming content + 'film', 'filme', 'video', 'videos', 'serie', 'serien', 'emby', 'media', + 'medienserver', 'streaming', 'film abspielen', 'wiedergabe', 'ansehen', + 'schauen', 'anschauen', 'wollen', 'möchte', 'komödie', 'drama', 'thriller', + 'action', 'abenteuer', 'dokumentation', 'doku', 'humor', + # English — video / streaming content + 'movie', 'movies', 'video', 'videos', 'series', 'show', 'shows', 'emby', + 'media', 'stream', 'streaming', 'watch', 'play', 'playback', 'tv', + }) + + + # ── Schemas for internal quick-command management tools. + # Injected into the per-turn tool list so the LLM can manage the quick-command + # store without an MCP server. Calls are intercepted in _dispatch_quick_tool. + _QUICK_TOOLS: list[dict] = [ + { + "type": "function", + "function": { + "name": "javis__quick_add", + "description": ( + "Add a new quick-response command. After successfully handling a " + "repeatable user request (device control, routines, etc.), call " + "this to save it for instant future execution without LLM " + "involvement. Do this silently — no announcement to the user. " + "Default match_mode is 'contains' for smart-home commands." + ), + "parameters": { + "type": "object", + "properties": { + "trigger": { + "type": "string", + "description": ( + "Short phrase that activates this command. For 'contains' " + "mode use key words (e.g. 'esszimmerlicht an'). For " + "'regex' mode supply a valid Python regex pattern." + ), + }, + "match_mode": { + "type": "string", + "enum": ["exact", "contains", "regex"], + "description": ( + "'contains': all trigger words must appear in input " + "(Alexa-style, recommended for smart-home). " + "'exact': normalised input must equal trigger. " + "'regex': trigger is a Python regex on raw input." + ), + }, + "actions": { + "type": "array", + "description": "Ordered MCP tool calls to execute.", + "items": { + "type": "object", + "properties": { + "tool": {"type": "string", "description": "Fully-qualified tool name, e.g. homey__control_zone."}, + "args": {"type": "object", "description": "Arguments dict for the tool."}, + }, + "required": ["tool", "args"], + }, + }, + }, + "required": ["trigger", "match_mode", "actions"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "javis__quick_list", + "description": "List all saved quick-response commands as JSON.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "javis__quick_update", + "description": "Update an existing quick command by id or trigger string.", + "parameters": { + "type": "object", + "properties": { + "id_or_trigger": {"type": "string", "description": "Command UUID or exact trigger string."}, + "trigger": {"type": "string", "description": "New trigger phrase (optional)."}, + "match_mode": {"type": "string", "enum": ["exact", "contains", "regex"], "description": "New match mode (optional)."}, + "actions": { + "type": "array", + "description": "Replacement action list (optional).", + "items": { + "type": "object", + "properties": { + "tool": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["tool", "args"], + }, + }, + }, + "required": ["id_or_trigger"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "javis__quick_delete", + "description": "Delete a quick command by id or trigger string.", + "parameters": { + "type": "object", + "properties": { + "id_or_trigger": {"type": "string", "description": "Command UUID or exact trigger string."}, + }, + "required": ["id_or_trigger"], + }, + }, + }, + ] + + # ── Schemas for internal timer tools. ───────────────────────────────── + # Injected into the per-turn tool list alongside _QUICK_TOOLS. + # Calls are intercepted in _dispatch_timer_tool. + _TIMER_TOOLS: list[dict] = [ + { + "type": "function", + "function": { + "name": "javis__timer_set", + "description": ( + "Set a countdown timer. When it fires Javis will announce it " + "via speech automatically. Use this for any request like " + "'set a timer for 5 minutes', 'timer 30 seconds', etc." + ), + "parameters": { + "type": "object", + "properties": { + "duration_seconds": { + "type": "integer", + "description": ( + "Timer duration in seconds. Convert natural language: " + "'5 minutes' → 300, '1 hour' → 3600, '90 seconds' → 90, " + "'1 hour 30 minutes' → 5400." + ), + }, + "label": { + "type": "string", + "description": ( + "Short human-friendly name for the timer " + "(e.g. 'pasta', '5-minute', 'egg'). " + "Omit to auto-generate from duration." + ), + }, + }, + "required": ["duration_seconds"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "javis__timer_cancel", + "description": "Cancel a running timer by its id.", + "parameters": { + "type": "object", + "properties": { + "timer_id": { + "type": "string", + "description": "Timer id returned by javis__timer_set (or from javis__timer_list).", + }, + }, + "required": ["timer_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "javis__timer_list", + "description": "List all active timers with their ids, labels, and seconds remaining.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + ] + + # ── Schema for the on-demand tool-loading meta-tool. ───────────────── + # Lets the model recover from keyword-routing misses: the runtime notice + # lists connected-but-unrouted integrations; calling this loads their + # tool schemas for the remaining iterations of the turn. + _META_TOOLS: list[dict] = [ + { + "type": "function", + "function": { + "name": "javis__load_tools", + "description": ( + "Load the tools of a connected integration that is not in " + "the current tool list. The RUNTIME STATUS section names " + "integrations available for loading. After this returns, " + "that integration's tools can be called directly." + ), + "parameters": { + "type": "object", + "properties": { + "server": { + "type": "string", + "description": ( + "Integration name exactly as listed in RUNTIME " + "STATUS (e.g. 'spotify', 'emby', 'homey')." + ), + }, + }, + "required": ["server"], + }, + }, + }, + ] + + # ── Schemas for on-demand memory access. ─────────────────────────── + # Recall/storage stay automatic; these let the model pull deeper or + # delete on explicit user request ("was weißt du über mich", "vergiss das"). + _MEMORY_TOOLS: list[dict] = [ + { + "type": "function", + "function": { + "name": "javis__memory_search", + "description": ( + "Search Javis's long-term memory on demand. Use when the " + "user asks what you know or remember about something, or " + "when the auto-recalled memories in the context lack a " + "fact you need. Storage remains automatic." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": ( + "Keywords to search for (device names, topics, " + "a person's name, …)." + ), + }, + }, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "javis__memory_forget", + "description": ( + "Delete a stored memory. Use when the user says 'vergiss " + "das', 'forget that', or retracts a stored fact. Pass the " + "exact memory key if known (from javis__memory_search), " + "or a phrase describing the fact." + ), + "parameters": { + "type": "object", + "properties": { + "key_or_query": { + "type": "string", + "description": ( + "Exact snake_case memory key, or a phrase to " + "match against stored facts." + ), + }, + }, + "required": ["key_or_query"], + }, + }, + }, + ] + + def _route_tools(self, tools: list[dict], user_text: str, + history: list[dict] | None = None) -> list[dict]: + """Return the focused subset of tools relevant to *user_text*. + + Each matching server's tools are included; time and fetch are always + included (tiny cost, high utility). When no server-specific keywords + match in the current message, recent conversation history (last 2 + messages) is scanned so that follow-ups and corrections like + "no you did not" / "das hat nicht geklappt" keep the right tools + available instead of falling back to the 3-tool conversational set. + """ + if not tools: + return tools + txt = user_text.lower() + wants_homey = any(k in txt for k in self._HOMEY_KEYWORDS) + wants_spotify = any(k in txt for k in self._SPOTIFY_KEYWORDS) + wants_files = any(k in txt for k in self._FILES_KEYWORDS) + wants_search = any(k in txt for k in self._SEARCH_KEYWORDS) + wants_slave = any(k in txt for k in self._SLAVE_KEYWORDS) + wants_emby = any(k in txt for k in self._EMBY_KEYWORDS) + + # Context fallback: if the current message carries no server-specific + # signal, scan the last 2 conversation turns (one user+assistant pair) + # so corrections / retries inherit the preceding exchange's context. + # Scanning only 2 messages (not 4) prevents older unrelated contexts + # (e.g. a Spotify session earlier in the conversation) from bleeding + # into a subsequent Homey-only retry turn. + if not (wants_homey or wants_spotify or wants_files or wants_search or wants_slave or wants_emby) and history: + recent = " ".join( + m.get("content", "") or "" + for m in history[-2:] + if isinstance(m.get("content"), str) + ).lower() + wants_homey = any(k in recent for k in self._HOMEY_KEYWORDS) + wants_spotify = any(k in recent for k in self._SPOTIFY_KEYWORDS) + wants_files = any(k in recent for k in self._FILES_KEYWORDS) + wants_search = any(k in recent for k in self._SEARCH_KEYWORDS) + wants_slave = any(k in recent for k in self._SLAVE_KEYWORDS) + wants_emby = any(k in recent for k in self._EMBY_KEYWORDS) + + include: set[str] = {'time__', 'fetch__', 'javis__quick_', 'javis__timer_', + 'javis__load_', 'javis__memory_'} + if wants_homey: + include.add('homey__') + if wants_spotify: + include.add('spotify__') + if wants_files: + include.add('filesystem__') + if wants_search: + include.add('duckduckgo__') + if wants_slave: + include.add('javis_slave__') + if wants_emby: + include.add('emby__') + + filtered = [ + t for t in tools + if any( + (t.get('function', {}).get('name', '') or '').startswith(p) + for p in include + ) + ] + if len(filtered) < len(tools): + log.info( + f"[tool-router] {len(filtered)}/{len(tools)} tools " + f"(homey={wants_homey}, spotify={wants_spotify}, " + f"files={wants_files}, search={wants_search}, slave={wants_slave}, emby={wants_emby})" + ) + return filtered or tools + + def _tool_availability_notice(self, tools: list[dict]) -> str: + """Build a per-turn system message describing live tool/MCP status. + + Derives the server list from *tools* so the notice accurately reflects + routing (tools may be a filtered subset of all available tools). + """ + snapshot = self.mcp.status_snapshot() + servers = snapshot.get("servers", []) + if not tools: + if not servers: + return ( + "RUNTIME STATUS: No MCP servers are connected and no tools are available. " + "You CANNOT control smart-home devices, read files, fetch URLs, or take any " + "external action this turn. If the user requests such an action, reply with " + "one short sentence stating that the required integration is not connected " + "(e.g. 'Smart-Home-Integration ist nicht verbunden.'). " + "Never claim an action succeeded. Never invent device names or results." + ) + return ( + "RUNTIME STATUS: MCP servers are configured but no tools are currently exposed. " + "Do not pretend to perform actions. Tell the user the integration is unavailable." + ) + + # Build server → tool-name mapping from the *actual* tools being sent. + # Using the tools list (not the snapshot) correctly reflects routing: + # only the servers whose schemas are included are listed here. + server_tools: dict[str, list[str]] = {} + for t in tools: + name = t.get("function", {}).get("name", "") + if "__" in name: + server_key, tool_name = name.split("__", 1) + else: + server_key, tool_name = "misc", name + server_tools.setdefault(server_key, []).append(tool_name) + + server_lines = [ + f"- {key}: {len(names)} tool(s) — {', '.join(names)}" + for key, names in sorted(server_tools.items()) + ] + # Integrations that are connected but whose tools were not routed into + # this turn — the model can load them on demand via javis__load_tools. + unloaded = sorted(set(self.mcp.servers) - set(server_tools)) + unloaded_note = "" + if unloaded: + unloaded_note = ( + "\n\nOther connected integrations (tools NOT loaded this turn): " + + ", ".join(unloaded) + + ". If the user's request needs one of these, call " + "javis__load_tools(server='') first — that integration's " + "tools become available immediately afterwards. Do NOT claim a " + "capability is missing without checking this list." + ) + return ( + "RUNTIME STATUS — AVAILABLE TOOLS (use these exact names for function calls):\n" + + "\n".join(server_lines) + + unloaded_note + + "\n\nIMPERATIVE: The tools above are your ONLY means of taking action. " + "To control a device, read a file, or perform any real-world action " + "you MUST issue a function call with one of the exact tool names listed above. " + "Do NOT write a result sentence before the tool call has returned. " + "Do NOT guess or invent tool names, device IDs, or results." + ) + + def _dispatch_timer_tool(self, name: str, args: dict) -> str: + """Handle internal javis__timer_* tool calls synchronously.""" + if name == "javis__timer_set": + raw = args.get("duration_seconds") + if raw is None: + return "ERROR: 'duration_seconds' is required." + try: + duration_s = int(raw) + except (TypeError, ValueError): + return f"ERROR: 'duration_seconds' must be an integer, got {raw!r}." + label = str(args.get("label") or "").strip() + try: + timer_id = self.timer_service.set_timer(label, duration_s) + except ValueError as exc: + return f"ERROR: {exc}" + return json.dumps({"ok": True, "timer_id": timer_id, "duration_s": duration_s, + "label": label or None}) + if name == "javis__timer_cancel": + timer_id = str(args.get("timer_id") or "").strip() + if not timer_id: + return "ERROR: 'timer_id' is required." + ok = self.timer_service.cancel_timer(timer_id) + if ok: + return json.dumps({"ok": True, "cancelled": timer_id}) + return f"ERROR: No active timer found with id {timer_id!r}." + if name == "javis__timer_list": + timers = self.timer_service.list_timers() + return json.dumps({"count": len(timers), "timers": timers}, ensure_ascii=False) + return f"ERROR: Unknown internal timer tool: {name!r}." + + async def _dispatch_memory_tool( + self, name: str, args: dict, speaker_name: "str | None" + ) -> str: + """Handle internal javis__memory_* tool calls.""" + if name == "javis__memory_search": + query = str(args.get("query") or "").strip() + if not query: + return "ERROR: 'query' is required." + return await self._memory_mw.search_memories_tool(query, speaker_name) + if name == "javis__memory_forget": + needle = str(args.get("key_or_query") or "").strip() + if not needle: + return "ERROR: 'key_or_query' is required." + return await self._memory_mw.forget_tool(needle, speaker_name) + return f"ERROR: Unknown internal memory tool: {name!r}." + + async def _on_timer_fire(self, timer_id: str, label: str) -> None: + """Called by TimerService when a timer expires; announces via TTS.""" + await self.bus.publish("timer_fired", {"id": timer_id, "label": label}) + announcement = f"Timer fertig! Dein {label} Timer ist abgelaufen." + log.info(f"[timer] announcing: {announcement}") + # Hold the turn lock for the whole announcement so we neither interrupt + # an in-progress turn nor let a new wake/PTT/text turn start (and have + # its state stomped by the trailing IDLE transition) mid-announcement. + if self._turn_lock.locked(): + log.debug("[timer] turn in progress — waiting before announcement") + async with self._turn_lock: + await self._play_timer_chime() + await self._speak(announcement) + await self.set_state(AgentState.IDLE) + + def _dispatch_quick_tool(self, name: str, args: dict) -> str: + """Handle internal javis__quick_* tool calls synchronously.""" + q = self.quick + if name == "javis__quick_add": + trigger = args.get("trigger", "") + mode = args.get("match_mode", "contains") + actions = args.get("actions", []) + if not trigger: + return "ERROR: 'trigger' is required." + if not actions: + return "ERROR: 'actions' must contain at least one entry." + try: + cmd_id = q.add(trigger, mode, actions) + except Exception as exc: + return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps({ + "ok": True, "id": cmd_id, "trigger": trigger, + "match_mode": mode, "action_count": len(actions), + }) + if name == "javis__quick_list": + cmds = q.list_all() + result = [ + { + "id": c.id, "trigger": c.trigger, "match_mode": c.match_mode, + "actions": [{"tool": a.tool, "args": a.args} for a in c.actions], + } + for c in cmds + ] + return json.dumps({"count": len(result), "commands": result}, ensure_ascii=False) + if name == "javis__quick_update": + id_or_trigger = args.get("id_or_trigger", "") + if not id_or_trigger: + return "ERROR: 'id_or_trigger' is required." + kwargs: dict = {k: v for k, v in args.items() if k != "id_or_trigger"} + try: + ok = q.update(id_or_trigger, **kwargs) + except Exception as exc: + return json.dumps({"ok": False, "error": str(exc)}) + if ok: + return json.dumps({"ok": True, "id_or_trigger": id_or_trigger}) + return f"ERROR: No command found matching {id_or_trigger!r}." + if name == "javis__quick_delete": + id_or_trigger = args.get("id_or_trigger", "") + if not id_or_trigger: + return "ERROR: 'id_or_trigger' is required." + ok = q.delete(id_or_trigger) + if ok: + return json.dumps({"ok": True, "deleted": id_or_trigger}) + return f"ERROR: No command found matching {id_or_trigger!r}." + return f"ERROR: Unknown internal tool: {name!r}." + + # ---------------- voice enrollment ---------------- + + async def start_enrollment(self, name: str) -> bool: + """Record ``config.speaker_id.min_enroll_clips`` clips and enroll *name*. + + Acquires ``_turn_lock`` so it cannot race with normal turns. Chimes + play between clips so the user knows when to speak. Returns *True* on + success, *False* if the mic is unavailable or a clip fails to capture. + Designed to be called from the GUI via ``asyncio.ensure_future()``. + """ + if self.mic is None or self.stt is None: + log.warning("[profiler] start_enrollment: mic/STT not available") + return False + if self._turn_lock.locked(): + await self.bus.publish("enroll_progress", { + "name": name, "clip": 0, "total": 0, + "error": "Ein anderer Vorgang läuft — bitte warten.", + }) + return False + + n = self.config.speaker_id.min_enroll_clips + clips: list[np.ndarray] = [] + async with self._turn_lock: + await self.set_state(AgentState.ENROLLING) + for i in range(n): + await self.bus.publish("enroll_progress", { + "name": name, "clip": i, "total": n, "recording": True, + }) + asyncio.ensure_future(self._play_wake_chime()) + audio = await record_utterance(self.config, self.mic, self.vad) + if audio is None or len(audio) < 1600: + await self.set_state(AgentState.IDLE) + await self.bus.publish("enroll_progress", { + "name": name, "clip": i, "total": n, + "error": "Kein Audio erkannt — Aufnahme abgebrochen.", + }) + return False + clips.append(audio) + await self.bus.publish("enroll_progress", { + "name": name, "clip": i + 1, "total": n, "recording": False, + }) + + sr = self.config.audio.sample_rate + await asyncio.to_thread(self.profiler.enroll, name, clips, sr) + await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)}) + await self.set_state(AgentState.IDLE) + log.info(f"[profiler] enrollment complete for '{name}'") + return True + + async def auto_enroll_confirm(self, key: str, name: str) -> bool: + """Finalize auto-enrollment using all clips buffered since the dialog opened. + + Called by the GUI when the user submits their name. *key* is kept for + API compatibility but is no longer used to look up audio — the full + clip buffer is consumed instead. + """ + clips = list(self._enroll_clip_buffer) + self._enroll_clip_buffer.clear() + self._enroll_ask_pending = False + + if not clips: + log.warning("[profiler] auto_enroll_confirm: clip buffer is empty") + return False + + sr = self.config.audio.sample_rate + existing_names = {p["name"] for p in self.profiler.list_profiles()} + if name in existing_names: + # Merge all buffered clips into the existing profile. + for clip in clips: + await asyncio.to_thread(self.profiler.refine, name, clip, sr) + log.info( + f"[profiler] merged {len(clips)} clip(s) into existing profile '{name}'" + ) + else: + # Build a new profile from all accumulated clips. + await asyncio.to_thread(self.profiler.enroll, name, clips, sr) + log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)") + await self.bus.publish("speaker_enrolled", {"name": name, "clip_count": len(clips)}) + return True + + async def _auto_enroll_speaker(self, audio: np.ndarray) -> None: + """Buffer *audio* from an unknown speaker and ask for a name (once). + + Subsequent unknown utterances while the dialog is open are silently + appended to the buffer so enrollment ultimately uses multiple clips. + """ + self._enroll_clip_buffer.append(audio) + if not self._enroll_ask_pending: + self._enroll_ask_pending = True + # Use a stable key so the GUI can match confirm callbacks. + key = str(int(time.monotonic() * 1000)) + log.info( + f"[profiler] unknown speaker — buffered clip 1, requesting name (key={key})" + ) + await self.bus.publish("speaker_ask_name", {"key": key}) + else: + log.info( + f"[profiler] unknown speaker — buffered clip {len(self._enroll_clip_buffer)}" + " (dialog already open)" + ) + + async def _bg_quick_suggest(self, user_text: str) -> None: # noqa: ARG002 + """Background quick-command optimizer — runs every N LLM turns. + + Sends the last few conversation turns to the LLM with a compact + system prompt focused exclusively on quick-command management. + Any javis__quick_add / update / delete calls it returns are executed + silently. No TTS, no spoken acknowledgement — only the GUI refreshes + (via the quick_updated event) if the list was modified. + """ + self._bg_quick_turn_count += 1 + if self._bg_quick_turn_count % self._BG_QUICK_EVERY_N != 1: + return + + try: + log.debug("[bg-quick] running background quick-command analysis") + + # Collect recent user/assistant turns (skip system + raw tool-call + # assistant messages whose content is usually None). + history = self.conv.messages() + recent = [ + m for m in history + if m.get("role") in ("user", "assistant") + and isinstance(m.get("content"), str) + and m["content"].strip() + ][-8:] # last ≈ 4 exchanges + + # Snapshot of the current quick-command list for context. + current = self.quick.list_all() + quick_json = json.dumps( + [ + { + "id": c.id, + "trigger": c.trigger, + "match_mode": c.match_mode, + "actions": [{"tool": a.tool, "args": a.args} for a in c.actions], + } + for c in current + ], + ensure_ascii=False, + ) + + system = ( + "You are a silent background optimizer for the Javis voice assistant.\n" + "Your sole task: keep the quick-command list useful and current.\n\n" + f"EXISTING QUICK COMMANDS: {quick_json}\n\n" + "INSTRUCTIONS:\n" + "1. Review the conversation for repeatable tool-based requests.\n" + "2. If a clear repeatable pattern exists and is NOT already covered, " + "call javis__quick_add (trigger, match_mode='contains', actions=[...]).\n" + "3. If an existing command is stale, broken, or improvable, " + "call javis__quick_update or javis__quick_delete.\n" + "4. If nothing needs changing, output only the word 'ok'. No explanation.\n" + "5. Do NOT speak. Do NOT announce changes. Only silent tool calls or 'ok'.\n" + "6. Never duplicate an existing trigger. Check the list first.\n" + "7. Use exact tool names visible in the conversation history.\n" + "8. CRITICAL — argument values (zone names, device IDs, capability strings, " + "etc.) MUST be copied verbatim from a successful tool-result that appears " + "in the conversation. Never invent, guess, or paraphrase argument values. " + "If you are not certain of the exact value, call javis__quick_delete " + "instead of updating with potentially wrong arguments.\n" + "9. Do NOT modify an existing quick command solely because the user " + "expressed dissatisfaction with its effect. Only update it when you can " + "copy a proven correct tool call from a successful result in this conversation.\n" + ) + + messages = [ + {"role": "system", "content": system}, + *recent, + {"role": "user", "content": "analyze"}, + ] + + response = await self.llm.chat(messages, tools=self._QUICK_TOOLS, stream=False) + + if not response.tool_calls: + log.debug("[bg-quick] no quick command changes needed") + return + + _MUTATORS = {"javis__quick_add", "javis__quick_update", "javis__quick_delete"} + mutated = False + for tc in response.tool_calls: + if not tc.name.startswith("javis__quick_"): + continue + log.info(f"[bg-quick] {tc.name} {tc.arguments}") + result = self._dispatch_quick_tool(tc.name, tc.arguments) + if tc.name in _MUTATORS and not result.startswith("ERROR:"): + mutated = True + log.info(f"[bg-quick] → {result[:120]}") + + if mutated: + await self.bus.publish("quick_updated", {}) + log.info("[bg-quick] quick commands updated") + + except Exception as exc: # noqa: BLE001 + log.debug(f"[bg-quick] background analysis skipped (non-critical): {exc}") + + async def _try_quick_response(self, text: str) -> bool: + """Execute a quick command if *text* matches one, bypassing the LLM. + + Returns True if a command matched and was executed (caller must skip + the LLM loop). Returns False when no pattern matches or the command + has no actions (empty-action commands are forwarded to the LLM so it + can diagnose or clean up the broken entry). + """ + cmd = self.quick.match(text) + if cmd is None: + return False + if not cmd.actions: + log.warning( + f"[quick] trigger={cmd.trigger!r} has no actions — " + "forwarding to LLM instead of silent no-op" + ) + return False + + # Guard: for "contains" mode, if the utterance is significantly longer + # than the trigger the user is almost certainly giving a meta-instruction + # ("when I say X, I want…") or a complaint ("warum geht X nicht") rather + # than issuing the command itself. +2 allows up to 2 filler words on top + # of the trigger (e.g. "schalte das Wohnzimmerlicht aus" = trigger 2 words + # + 2 fillers = 4 words total), while reliably rejecting 5+ word diagnostic + # questions that also happen to contain the trigger words. + if cmd.match_mode == "contains": + trigger_words = len(self.quick._normalize(cmd.trigger).split()) + utterance_words = len(self.quick._normalize(text).split()) + if utterance_words > trigger_words + 2: + log.info( + f"[quick] skipping trigger={cmd.trigger!r} — utterance " + f"({utterance_words} words) far exceeds trigger ({trigger_words} words); " + f"forwarding to LLM (likely a meta-instruction)" + ) + return False + + log.info(f"[quick] matched trigger={cmd.trigger!r} id={cmd.id} mode={cmd.match_mode}") + await self.bus.publish("quick_match", {"id": cmd.id, "trigger": cmd.trigger}) + + # Record the user utterance in conversation history so follow-up turns + # ("that didn't work", "undo it") have context about what just ran. + self.conv.add_user(text) + + errors: list[str] = [] + for action in cmd.actions: + # Publish tool_call so the Status tab shows the in-flight action. + await self.bus.publish("tool_call", {"name": action.tool, "args": action.args}) + try: + # Intercept internal javis__ tools so quick actions can reference + # them without going through MCP (which doesn't know about them). + if action.tool.startswith("javis__timer_"): + result = self._dispatch_timer_tool(action.tool, action.args) + elif action.tool.startswith("javis__quick_"): + result = self._dispatch_quick_tool(action.tool, action.args) + else: + # Same reasoning as the main loop's _call_mcp: do not bypass + # the MCP server's approval='ask' policy just because this + # came from a pre-learned quick command. + result = await self.mcp.call(action.tool, action.args) + ok = not result.startswith("ERROR:") + if not ok: + errors.append(f"{action.tool}: {result[:200]}") + log.info(f"[quick] {action.tool}: {result[:120]}") + await self.bus.publish("tool_result", { + "name": action.tool, "latency_s": 0.0, + "ok": ok, "preview": result[:200], + }) + except Exception as exc: + errors.append(f"{action.tool}: {exc}") + log.warning(f"[quick] {action.tool} failed: {exc}") + await self.bus.publish("tool_result", { + "name": action.tool, "latency_s": 0.0, + "ok": False, "preview": str(exc)[:200], + }) + + if errors: + err_summary = "; ".join(e[:120] for e in errors[:3]) + await self.bus.publish("error", {"where": "quick", "message": err_summary}) + log.warning(f"[quick] command failed — forwarding to LLM for repair: {err_summary}") + + # Build repair directive injected into the system message so the + # LLM treats it as a high-priority instruction rather than its own + # prior assistant turn (avoids the "Was möchtest du wissen?" drift). + failed_details = "\n".join( + f" - {a.tool}({json.dumps(a.args, ensure_ascii=False)}) → {e}" + for a, e in zip(cmd.actions, errors) + ) + repair_directive = ( + f"REPAIR TASK: The quick command '{cmd.trigger}' (id: {cmd.id}) " + f"just failed with these errors:\n{failed_details}\n" + f"You MUST: (1) call javis__quick_update with id='{cmd.id}' and " + f"corrected actions, then (2) call the correct tool to fulfil " + f"the user's original request. Do NOT ask the user anything." + ) + + # Run the LLM so it can repair the command and complete the turn. + # TTS is intentionally skipped — the repair result is shown in the + # GUI only; raw error context should not be read aloud. + cmd_updated_at_before = cmd.updated_at + hist_len_before = len(self.conv.history) + turn_ctx = TurnContext(turn_id=f"qfix_{cmd.id[:8]}") + await run_start(self._middleware, turn_ctx) + try: + reply = await self._llm_loop(turn_ctx, extra_system=repair_directive) + if reply: + await self.bus.publish("assistant_text", reply) + await run_finish(self._middleware, turn_ctx, text, reply) + await turn_ctx.flush() + # Auto-patch: if the LLM skipped javis__quick_update, + # extract the successful tool calls it DID make and repair + # the command automatically so the next invocation works. + current_cmd = self.quick._commands.get(cmd.id) + if current_cmd and current_cmd.updated_at == cmd_updated_at_before: + new_msgs = list(self.conv.history)[hist_len_before:] + new_actions = self._extract_repair_actions(new_msgs) + if new_actions: + self.quick.update(cmd.id, actions=new_actions) + await self.bus.publish("quick_updated", {}) + log.info( + f"[quick] auto-patched command {cmd.id!r} with " + f"{len(new_actions)} action(s) from repair turn" + ) + else: + # Repair loop produced no usable tool calls — the + # command is stuck broken. Delete it so the next + # invocation falls through to the LLM (which can + # execute it correctly) and bg-quick can recreate + # a valid command from the resulting conversation. + self.quick.delete(cmd.id) + await self.bus.publish("quick_updated", {}) + log.warning( + f"[quick] deleted broken command {cmd.id!r} — " + f"repair loop produced no successful tool calls; " + f"will be recreated by bg-quick after next LLM turn" + ) + else: + await run_abort(self._middleware, turn_ctx, "empty_repair_reply") + except Exception as exc: # noqa: BLE001 + await run_error(self._middleware, turn_ctx, exc) + log.error(f"[quick] LLM repair loop failed: {exc}") + else: + # Brief descending chime confirms the command fired without + # cluttering the conversation with a spoken acknowledgement. + await self._play_confirmation_chime() + # Record a synthetic assistant turn so the LLM has context if the + # user immediately follows up ("that was wrong", "undo it"). + self.conv.add_assistant("OK.") + + return True + + @staticmethod + def _extract_repair_actions(new_messages: list[dict]) -> list[dict[str, Any]]: + """Scan conversation messages added during a repair loop and return + QuickAction dicts for every successful non-quick tool call found.""" + # Collect all successful tool results keyed by tool name (first occurrence wins). + tool_results: dict[str, str] = {} + for msg in new_messages: + if msg.get("role") == "tool": + name = msg.get("name", "") + if name not in tool_results: + tool_results[name] = msg.get("content", "") + + actions: list[dict[str, Any]] = [] + for msg in new_messages: + if msg.get("role") != "assistant": + continue + for tc_entry in msg.get("tool_calls") or []: + fn = tc_entry.get("function", {}) + name = fn.get("name", "") + if not name or name.startswith("javis__quick_"): + continue + args = fn.get("arguments") or {} + if not isinstance(args, dict): + continue + # Only include tool calls whose result is not an error. + result = tool_results.get(name, "") + if result.startswith("ERROR:"): + continue + actions.append({"tool": name, "args": args}) + return actions + + # ── Cross-turn tool result cache helpers ───────────────────────────────── + # Patterns for read-like tools that are safe to cache across turns. + # Write-like tools (control, set, play, delete, …) bypass the cache and + # also invalidate any cached reads for the same server prefix. + _CACHE_READ_PREFIXES = ( + "__get_", "__find_", "__list_", "__search", + "time__", "duckduckgo__", + ) + _CACHE_WRITE_PATTERNS = ( + "__control_", "__set_", "__play", "__stop", "__delete", + "__add_", "__update_", "__create_", "__remove_", + ) + + def _is_cacheable(self, tool_name: str) -> bool: + return any(p in tool_name for p in self._CACHE_READ_PREFIXES) + + def _invalidate_cache_for_write(self, tool_name: str) -> None: + """Evict read-cache entries for the same server when a write tool runs.""" + prefix = tool_name.split("__")[0] + "__" if "__" in tool_name else "" + if not prefix: + return + evict = [k for k in self._tool_cache if k.startswith(prefix)] + for k in evict: + del self._tool_cache[k] + if evict: + log.debug(f"[tool-cache] evicted {len(evict)} entr(ies) after write to '{tool_name}'") + + def _tool_cache_get(self, call_key: str) -> str | None: + """Return a cached result if it exists and is within TTL, else None.""" + ttl = self.config.agent.tool_cache_ttl_seconds + if ttl <= 0: + return None + entry = self._tool_cache.get(call_key) + if entry is None: + return None + result, ts = entry + age = time.monotonic() - ts + if age > ttl: + del self._tool_cache[call_key] + return None + log.debug(f"[tool-cache] hit '{call_key.split(':')[0]}' age={age:.0f}s") + return result + + # Hard cap on cached entries — read-tool argument combinations are + # unbounded over a long-running session (different queries/zones/etc.), + # and TTL alone only evicts a key when it happens to be looked up again. + # Without a cap, stale entries for keys the model never repeats would + # accumulate in memory for the lifetime of the process. + _TOOL_CACHE_MAX_ENTRIES = 200 + + def _tool_cache_put(self, call_key: str, result: str) -> None: + # Dict preserves insertion order — evict the oldest entries first + # (simple FIFO, not true LRU, but sufficient to bound memory here). + if len(self._tool_cache) >= self._TOOL_CACHE_MAX_ENTRIES: + self._tool_cache.pop(next(iter(self._tool_cache))) + self._tool_cache[call_key] = (result, time.monotonic()) + + async def _llm_loop(self, turn_ctx: TurnContext, extra_system: str | None = None, + tts_feed: "asyncio.Queue | None" = None) -> str: + max_iters = self.config.agent.max_llm_iterations + # A "Deny" only applies to the turn it was clicked in. + self.mcp.clear_denials() + all_tools = ( + self.mcp.ollama_tools() + + self._QUICK_TOOLS + + self._TIMER_TOOLS + + self._META_TOOLS + + self._MEMORY_TOOLS + ) + + # Route tools to the relevant subset based on the last user message. + # This prevents CUDA OOM from the 65-tool full-schema payload. + msgs = self.conv.messages() + last_user_text = next( + (m["content"] for m in reversed(msgs) if m.get("role") == "user"), + "", + ) + tools = self._route_tools(all_tools, last_user_text, history=msgs) + runtime_notice = self._tool_availability_notice(tools) + + # Per-turn dedup cache: maps call_key → last result string. + # Prevents qwen2.5 from calling the same tool with identical args in a + # loop; the second call returns the cached result immediately. + _call_results: dict[str, str] = {} + + # Incremental sentence buffer for the streaming-TTS pipeline. + # Tokens are accumulated here; complete sentences are pushed to + # tts_feed so synthesis can start while the LLM is still generating. + _stream_buf: list[str] = [] + _streaming_tts_used: bool = False + + # ── Qwen3 block stripping ──────────────────────────────────── + # When thinking_mode=True (or as a safety net even when False), Qwen3 + # may emit … reasoning tokens before the visible answer. + # We suppress those tokens from TTS and the UI delta stream. + # Tags can arrive split across multiple deltas, so we keep an overflow + # buffer (_think_tail) for the last (tag_len - 1) bytes. + _THINK_OPEN = "" + _THINK_CLOSE = "" + _THINK_TAG_MAX = max(len(_THINK_OPEN), len(_THINK_CLOSE)) # 8 + _in_think: bool = False + _think_tail: str = "" # uncommitted chars that might start a tag + + def _filter_think(raw: str) -> str: + """Strip … content from *raw*, handling split tags.""" + nonlocal _in_think, _think_tail + raw = _think_tail + raw + _think_tail = "" + visible_parts: list[str] = [] + while raw: + if _in_think: + ci = raw.find(_THINK_CLOSE) + if ci == -1: + # Inside think block; hold the tail in case is split. + guard = min(len(_THINK_CLOSE) - 1, len(raw)) + _think_tail = raw[-guard:] + break + raw = raw[ci + len(_THINK_CLOSE):] + _in_think = False + else: + oi = raw.find(_THINK_OPEN) + if oi == -1: + # No tag starting here; hold a small tail guard. + guard = min(_THINK_TAG_MAX - 1, len(raw)) + visible_parts.append(raw[:-guard] if guard else raw) + _think_tail = raw[-guard:] if guard else "" + break + visible_parts.append(raw[:oi]) + raw = raw[oi + len(_THINK_OPEN):] + _in_think = True + return "".join(visible_parts) + + # Stream content deltas for a snappier UI (typing effect). Tool calls + # still arrive in the final assembled response. + async def _on_delta(delta: str) -> None: + nonlocal _streaming_tts_used + visible = _filter_think(delta) + if not visible: + return + await self.bus.publish("assistant_delta", visible) + if tts_feed is None: + return + _stream_buf.append(visible) + accumulated = "".join(_stream_buf) + parts = SENTENCE_BOUNDARY.split(accumulated) + if len(parts) > 1: + # Everything except the trailing fragment is a complete sentence. + complete = " ".join(p.strip() for p in parts[:-1] if p.strip()) + if complete: + await tts_feed.put(complete) + _streaming_tts_used = True + _stream_buf.clear() + if parts[-1].strip(): + _stream_buf.append(parts[-1]) + + # Transient nudge injected into the *messages* list (not into conv) so + # that phantom turns don't pollute conversation history. + nudge: str | None = None + + # Track consecutive empty responses to detect stuck-model patterns early + # and trigger force_finalize to break the loop. + empty_response_count: int = 0 + max_empty_responses: int = 2 # Allow at most 2 consecutive empties before forcing finalize + + # When True the next model call runs WITHOUT tools (streaming) so the + # model must produce a final text answer instead of more tool calls. + # Set when an iteration consists exclusively of duplicate tool calls + # (the model is cycling); the last iteration always finalizes. + force_finalize: bool = False + + for i in range(max_iters): + # Final-answer round: drop tools so the model must summarise the + # outcome for the user. Streaming is safe without tools and gives + # incremental TTS for the closing reply. Guard max_iters > 1 so a + # single-iteration config still gets one tool-capable round. + finalize = force_finalize or ( + bool(tools) and max_iters > 1 and i == max_iters - 1 + ) + + await self.bus.publish("llm_status", { + "phase": "calling", "iter": i + 1, "max_iter": max_iters, + "model": self.config.llm.model, + "tools": 0 if finalize else len(tools), + }) + t0 = time.monotonic() + + # Build the message list explicitly so the runtime notice always + # reaches the model. Most models (qwen2.5, llama3, mistral) only + # honour the *first* system-role entry — a second system message is + # silently dropped or mis-classified, so the model never sees the + # live tool list and hallucinates text instead of calling tools. + # Merging into a single system message is the only reliable approach. + history = self.conv.messages() + system_msg = history[0] + rest = history[1:] + merged_system_content = system_msg["content"].rstrip() + "\n\n" + runtime_notice + if extra_system: + merged_system_content = merged_system_content + "\n\n" + extra_system + if finalize and tools: + merged_system_content += ( + "\n\nFINAL ROUND: Tool calls are no longer available. " + "Answer the user directly now in one or two short sentences, " + "summarising what was done — or what failed and why — based on " + "the tool results above. Do not promise further actions." + ) + messages = [{"role": "system", "content": merged_system_content}, *rest] + + # Inject a pending nudge transiently (NOT into conv) so that empty + # responses cause a clean retry without phantom history turns. + if nudge: + messages = messages + [ + {"role": "assistant", "content": ""}, + {"role": "user", "content": nudge}, + ] + nudge = None + + # ── middleware: inject memory / transform messages ───────────────── + turn_ctx.iteration = i + messages = await run_before_llm(self._middleware, turn_ctx, messages, i) + + # Log context headroom so runaway context growth is visible early. + ctx_tokens = self.conv.approx_tokens + ctx_limit = self.config.llm.num_ctx + if ctx_tokens > ctx_limit * 0.80: + log.warning( + f"[ctx] {ctx_tokens}/{ctx_limit} tokens " + f"({ctx_tokens / ctx_limit:.0%}) — approaching limit" + ) + else: + log.debug(f"[ctx] {ctx_tokens}/{ctx_limit} tokens") + + # Ollama's streaming API drops or mangles `tool_calls` on many models + # (qwen2.5, llama3.1, mistral). Tool calling is only reliable with + # stream=false. So: when tools are available, use non-streaming; on + # the finalize round (tools withheld), stream for snappy TTS. + if tools and not finalize: + response = await self.llm.chat(messages, tools=tools, stream=False) + log.info( + f"[diag] iter={i+1} tool_calls={len(response.tool_calls)} " + f"content={repr((response.content or '')[:80])}" + ) + # Surface silent tool-schema fallbacks: the backend answered + # WITHOUT tools, so no action can have happened this call. + if response.tools_dropped: + log.warning( + "[llm_loop] backend dropped tool schemas — model " + "answered without tools this call" + ) + await self.bus.publish("llm_status", { + "phase": "tools_dropped", "iter": i + 1, + "max_iter": max_iters, + }) + if "TOOLS UNAVAILABLE NOTICE" not in runtime_notice: + runtime_notice += ( + "\n\nTOOLS UNAVAILABLE NOTICE: the model backend " + "rejected the tool schemas on a previous step of this " + "turn. If an action could not actually be performed, " + "say so honestly — never claim success without a " + "tool result." + ) + # ── text-mode tool call fallback ────────────────────────────── + # Detect tool calls the model emitted as plain text (e.g. + # "-server__tool {json}") and promote them to structured calls + # so the agentic loop executes them correctly. + if not response.tool_calls and response.content: + promoted, clean_content = _parse_text_tool_calls(response.content) + if promoted: + log.warning( + f"[text-tc-fallback] {len(promoted)} tool call(s) emitted " + f"as plain text — promoting to structured calls" + ) + response = ChatResponse( + content=clean_content, + tool_calls=promoted, + raw=response.raw, + ) + if response.content and not response.tool_calls: + # Surface the full reply as one delta so the UI updates. + # Strip … reasoning blocks first — they are + # removed from history/TTS later but must not leak to the UI. + visible = re.sub( + r".*?", "", response.content, flags=re.DOTALL + ).strip() + if visible: + await self.bus.publish("assistant_delta", visible) + else: + response = await self.llm.chat_streaming( + messages, + tools=None, + on_delta=_on_delta, + ) + # Flush any sentence fragment still in the buffer after the + # stream ends (text that never reached a sentence boundary). + if tts_feed is not None and _stream_buf: + remaining = "".join(_stream_buf).strip() + if remaining: + await tts_feed.put(remaining) + _streaming_tts_used = True + _stream_buf.clear() + latency = time.monotonic() - t0 + await self.bus.publish("llm_status", { + "phase": "response", "iter": i + 1, "max_iter": max_iters, + "model": self.config.llm.model, "latency_s": latency, + "tool_calls": len(response.tool_calls), + "content_len": len(response.content or ""), + }) + + if response.tool_calls: + # Record assistant tool-call message + self.conv.add_assistant( + response.content, + tool_calls=[ + {"function": {"name": tc.name, "arguments": tc.arguments}} + for tc in response.tool_calls + ], + ) + + # ── Routing recovery ───────────────────────────────────────── + # The model may call a tool it knows from conversation history + # whose schemas were NOT routed into this turn (keyword miss). + # The dispatch below still works, but follow-up calls would use + # guessed argument signatures. Expand the toolset with that + # server's schemas for subsequent iterations. + routed_names = { + t.get("function", {}).get("name", "") for t in tools + } + unrouted_servers = { + tc.name.split("__", 1)[0] + for tc in response.tool_calls + if "__" in tc.name and tc.name not in routed_names + } & set(self.mcp.servers) + if unrouted_servers: + extra_tools = [ + t for t in all_tools + if (name := t.get("function", {}).get("name", "")) + and name not in routed_names + and name.split("__", 1)[0] in unrouted_servers + ] + if extra_tools: + tools = tools + extra_tools + runtime_notice = self._tool_availability_notice(tools) + log.info( + f"[tool-router] expanded with {len(extra_tools)} tool(s) " + f"from un-routed server(s) {sorted(unrouted_servers)}" + ) + + # ── Pre-flight: serial middleware + dedup checks ────────────── + # run_before_tool must run serially because AbortTurnDecision + # halts the whole turn. We resolve skip/dedup/quick calls here + # and record None for calls that still need an MCP round-trip so + # they can be dispatched in parallel afterwards. + # + # Slot layout: _resolved[i] = (tc, decision, result, latency) + # A None placeholder marks a pending MCP call. + _resolved: list = [] + _pending: list = [] # (slot_index, tc, call_key) + _abort_early: str | None = None + _dup_hits: int = 0 # loop-guard hits in this batch + + for tc in response.tool_calls: + log.info(f"tool call: {tc.name} {tc.arguments}") + await self.bus.publish("tool_call", {"name": tc.name, "args": tc.arguments}) + + # ── middleware: on_before_tool ──────────────────────────── + decision = await run_before_tool( + self._middleware, turn_ctx, tc.name, tc.arguments + ) + if isinstance(decision, AbortTurnDecision): + log.warning(f"[middleware] turn aborted: {decision.reason}") + await run_abort(self._middleware, turn_ctx, decision.reason) + _abort_early = decision.reason or ( + "Anfrage durch Middleware abgebrochen. / " + "Turn aborted by middleware." + ) + break + if isinstance(decision, TransformArgsDecision): + tc = ToolCall( + name=tc.name, + arguments=decision.args, + call_id=tc.call_id, + ) + + if isinstance(decision, SkipToolDecision): + log.debug(f"[middleware] skip tool {tc.name} (cache hit)") + _resolved.append((tc, decision, decision.result, 0.0)) + continue + + # ── loop-guard: deduplicate identical calls ─────────────── + call_key = ( + f"{tc.name}:" + f"{json.dumps(tc.arguments, sort_keys=True, ensure_ascii=False)}" + ) + if call_key in _call_results: + log.warning( + f"[loop-guard] duplicate call {tc.name} " + f"— reusing cached result, skipping MCP" + ) + _dup_hits += 1 + _resolved.append((tc, decision, ( + "[Already called with identical arguments this turn. " + "Cached result:]\n" + _call_results[call_key] + ), 0.0)) + continue + + if tc.name == "javis__load_tools": + server = str(tc.arguments.get("server") or "").strip() + if server in self.mcp.servers: + routed_now = { + t.get("function", {}).get("name", "") for t in tools + } + extra_tools = [ + t for t in all_tools + if (n := t.get("function", {}).get("name", "")) + and n not in routed_now + and n.split("__", 1)[0] == server + ] + if extra_tools: + tools = tools + extra_tools + runtime_notice = self._tool_availability_notice(tools) + names = [ + t["function"]["name"] for t in extra_tools + ] + result = json.dumps( + {"ok": True, "loaded": names}, + ensure_ascii=False, + ) + log.info( + f"[tool-router] javis__load_tools loaded " + f"{len(extra_tools)} tool(s) from {server!r}" + ) + else: + result = json.dumps({ + "ok": True, "loaded": [], + "note": f"tools of '{server}' are already loaded", + }) + else: + result = ( + f"ERROR: unknown integration {server!r}. " + f"Connected: {', '.join(sorted(self.mcp.servers))}" + ) + _call_results[call_key] = result + _resolved.append((tc, decision, result, 0.0)) + continue + + if tc.name.startswith("javis__memory_"): + result = await self._dispatch_memory_tool( + tc.name, tc.arguments, turn_ctx.speaker_name + ) + _call_results[call_key] = result + _resolved.append((tc, decision, result, 0.0)) + continue + + if tc.name.startswith("javis__timer_"): + result = self._dispatch_timer_tool(tc.name, tc.arguments) + _call_results[call_key] = result + _resolved.append((tc, decision, result, 0.0)) + continue + + if tc.name.startswith("javis__quick_"): + result = self._dispatch_quick_tool(tc.name, tc.arguments) + _call_results[call_key] = result + _QUICK_MUTATORS = { + "javis__quick_add", + "javis__quick_update", + "javis__quick_delete", + } + if tc.name in _QUICK_MUTATORS: + await self.bus.publish("quick_updated", {}) + _resolved.append((tc, decision, result, 0.0)) + continue + + # ── cross-turn cache check ──────────────────────────────── + cached = self._tool_cache_get(call_key) + if cached is not None: + _call_results[call_key] = cached + _resolved.append((tc, decision, cached, 0.0)) + continue + + # Invalidate stale reads if this is a write tool. + if any(p in tc.name for p in self._CACHE_WRITE_PATTERNS): + self._invalidate_cache_for_write(tc.name) + + # Needs MCP — reserve a slot; dispatch later in parallel. + _pending.append((len(_resolved), tc, call_key)) + _resolved.append(None) # placeholder + + if _abort_early is not None: + if tts_feed is not None: + if not _streaming_tts_used and _abort_early: + await tts_feed.put(_abort_early) + await tts_feed.put(None) + return _abort_early + + # ── Parallel MCP dispatch ───────────────────────────────────── + # When multiple tools need MCP, fire them concurrently and fill + # the pre-reserved slots so results stay in original call order. + if _pending: + async def _call_mcp(tc: ToolCall) -> tuple[str, float]: + tt = time.monotonic() + # NOTE: pre_approved is intentionally NOT set here. + # PrivilegeMiddleware (run above, via run_before_tool) only + # enforces the per-speaker ACL — it is a different gate from + # the MCP server's own approval='ask' policy, which still + # needs the interactive confirmation dialog for servers like + # filesystem/spotify. _allow_cache remembers a grant for the + # rest of the process, so this only prompts once per tool. + r = await self.mcp.call(tc.name, tc.arguments) + return r, time.monotonic() - tt + + mcp_results = await asyncio.gather( + *[_call_mcp(entry[1]) for entry in _pending], + return_exceptions=True, + ) + for (slot, tc, call_key), outcome in zip(_pending, mcp_results): + if isinstance(outcome, BaseException): + log.error(f"MCP call {tc.name!r} raised: {outcome}") + result = f"ERROR: {tc.name} failed: {outcome}" + latency = 0.0 + else: + result, latency = outcome + _call_results[call_key] = result + # Populate cross-turn cache for read-like tools. + if self._is_cacheable(tc.name) and not result.startswith("ERROR:"): + self._tool_cache_put(call_key, result) + _resolved[slot] = (tc, None, result, latency) + + # ── Post-processing: publish + update conv (in original order) ─ + for entry in _resolved: + if entry is None: + continue # shouldn't happen + tc, decision, result, tool_latency = entry + ok = not result.startswith("ERROR:") + await self.bus.publish("tool_result", { + "name": tc.name, "latency_s": tool_latency, + "ok": ok, + "preview": result[:200], + }) + log.info(f"tool result ({tc.name}, {tool_latency:.2f}s): {result[:200]}") + + # Cap the stored result to prevent context window bloat; + # JSON payloads are trimmed at element boundaries. + stored_result = _truncate_tool_result(result) + self.conv.add_tool_result(tc.name, stored_result, tc.call_id) + + # ── middleware: on_after_tool ───────────────────────────── + await run_after_tool( + self._middleware, turn_ctx, tc.name, tc.arguments, + result, ok, tool_latency, + ) + + # ── Loop detection ──────────────────────────────────────────── + # Every call in this batch was a duplicate of an earlier call + # this turn — the model is cycling. Force a final no-tools + # answer round instead of burning the remaining iterations on + # identical results. + if _dup_hits and _dup_hits == len(response.tool_calls): + log.warning( + f"[loop-guard] iteration {i + 1} produced only duplicate " + f"tool calls — forcing final answer round" + ) + force_finalize = True + continue + + raw_content = (response.content or "").strip() + # Strip any … blocks from non-streaming responses + # (tool-call turns use non-streaming chat; also covers the case + # where think=True is configured and the model is not streamed). + text = re.sub(r".*?", "", raw_content, flags=re.DOTALL).strip() + + # Model returned neither content nor tool calls — this happens when + # temperature=0 causes the model to get stuck (e.g. recalled + # memories are already injected AND memory tools are available, + # creating conflicting signals). Retry with nudge messages that get + # progressively more directive; after max_empty_responses, force + # finalize (drop tools) to break the stuck loop. + if not text: + empty_response_count += 1 + if empty_response_count == 1: + # First empty response: try a stronger directive nudge + if i < max_iters - 1: + log.warning( + f"[llm_loop] iter={i+1}: model returned empty content with " + f"no tool calls — setting nudge and retrying (attempt {empty_response_count})" + ) + # More explicit nudge that forces a decision: either use a tool or answer directly + nudge = ( + "WICHTIG: Du musst antworten. / CRITICAL: You must respond. " + "Antworte direkt mit deiner Antwort oder nutze ein Tool. " + "Answer directly with your response or use a tool." + ) + continue + elif empty_response_count >= max_empty_responses: + # Multiple consecutive empty responses: model is stuck + # Force finalize (drop tools and stream) to get *any* output + log.warning( + f"[llm_loop] iter={i+1}: {empty_response_count} consecutive empty responses — " + f"forcing finalize round to recover from stuck state" + ) + force_finalize = True + empty_response_count = 0 # Reset counter after forcing finalize + continue + elif i < max_iters - 1: + # Second+ empty response: try again with a lighter nudge + log.warning( + f"[llm_loop] iter={i+1}: model still empty on retry " + f"(attempt {empty_response_count}) — retrying" + ) + nudge = ( + "Bitte antworte jetzt. / Please respond now. " + "Nur einen kurzen Satz. / Just one short sentence." + ) + continue + log.warning( + f"[llm_loop] iter={i+1}: empty response on final iteration " + f"— falling through to spoken fallback" + ) + break + else: + # Successful response: reset empty counter + empty_response_count = 0 + + if text: + self.conv.add_assistant(text) + log.info(f"assistant: {text!r}") + await self.bus.publish("llm_status", {"phase": "done", "iter": i + 1}) + if tts_feed is not None: + # Non-streaming path (tool turns): feed the full reply so + # _speak_stream can process it. Streaming path: all sentences + # were already pushed incrementally; only send the sentinel. + if not _streaming_tts_used and text: + await tts_feed.put(text) + await tts_feed.put(None) + return text + + _loop_fallback = ( + "Entschuldigung, ich stecke in einer Schleife fest. " + "Bitte formuliere die Anfrage neu. / " + "Sorry, I'm stuck in a loop — try rephrasing the request." + ) + log.warning("LLM loop hit max iterations") + await self.bus.publish("llm_status", {"phase": "max_iter"}) + if tts_feed is not None: + if not _streaming_tts_used: + await tts_feed.put(_loop_fallback) + await tts_feed.put(None) + return _loop_fallback + + # ---------------- TTS ---------------- + + async def _speak_stream(self, tts_feed: "asyncio.Queue[str | None]") -> None: + """TTS playback driven by a streaming sentence feed from *_llm_loop*. + + For conversational turns (no MCP tools), sentences arrive in *tts_feed* + in real-time as the LLM generates — first audio starts as soon as the + first sentence is synthesized, while the LLM is still streaming the + rest of the reply. For tool-calling turns, the full reply arrives as a + single feed item after all tool calls complete (same latency as before, + different code path). + """ + if self.tts is None: + # Drain feed so _llm_loop is never blocked on a full queue. + while True: + item = await tts_feed.get() + if item is None: + break + await self.bus.publish("tts_status", { + "phase": "skipped", "voice": self.config.tts.default_voice, + "reason": "TTS disabled", + }) + return + + profile = self.voices.get_or_create_default(self.config.tts.default_voice) + if not profile.reference_clips: + log.warning( + f"voice '{profile.name}' has no reference clips. Add WAVs to " + f"{profile.dir}/refs/ to enable speech." + ) + # Drain feed. + while True: + item = await tts_feed.get() + if item is None: + break + await self.bus.publish("tts_status", { + "phase": "skipped", "voice": profile.name, + "reason": "no reference clips", + }) + await self.bus.publish("error", { + "where": "tts", + "message": ( + f"Stimmprofil '{profile.name}' hat keine Referenzklips — " + f"Voices-Tab öffnen und WAV-Dateien in refs/ ablegen." + ), + }) + return + + self._cancel_playback.clear() + _state_set = False # switch to SPEAKING on first played chunk + + # Synthesis → playback pipeline identical to _speak, but the producer + # reads sentence fragments from tts_feed instead of a pre-built list. + synth_q: asyncio.Queue = asyncio.Queue(maxsize=3) + + async def _stream_producer() -> None: + idx = 0 + while True: + if self._cancel_playback.is_set(): + break + raw = await tts_feed.get() + if raw is None: + break + # Normalise and re-chunk the fragment (applies markdown cleanup + # and merges short pieces up to _SPEECH_CHUNK_MAX_CHARS). + for chunk in _group_speech_chunks(raw): + if self._cancel_playback.is_set(): + break + idx += 1 + t0 = time.monotonic() + try: + result = await self.tts.synthesize(chunk, profile) + except Exception as e: # noqa: BLE001 + log.error(f"[stream-tts] synthesis failed: {e}") + await self.bus.publish("tts_status", { + "phase": "error", "idx": idx, "error": str(e), + }) + continue + synth_latency = time.monotonic() - t0 + await synth_q.put((idx, chunk, result, synth_latency)) + await synth_q.put(None) + + prod_task = asyncio.create_task(_stream_producer()) + _out_stream = None + + def _write_pcm_stream(audio: np.ndarray) -> bool: + samples = np.ascontiguousarray(np.asarray(audio, dtype=np.float32)) + if samples.ndim == 1: + samples = samples[:, np.newaxis] + if _out_stream is not None and samples.shape[1] < _out_stream.channels: + samples = np.repeat(samples, _out_stream.channels, axis=1) + i, blk = 0, _out_stream.blocksize # type: ignore[union-attr] + while i < len(samples): + if self._cancel_playback.is_set(): + return True + end = min(i + blk, len(samples)) + _out_stream.write(samples[i:end]) # type: ignore[union-attr] + i = end + return False + + played = 0 + try: + self._start_barge_in() + while True: + item = await synth_q.get() + if item is None: + break + if self._cancel_playback.is_set(): + await self.bus.publish("tts_status", { + "phase": "cancelled", "idx": item[0], + }) + break + idx, sentence, result, synth_latency = item + + if _out_stream is None: + import sounddevice as sd + from ..audio_io.stream import _output_channels + _out_stream = sd.OutputStream( + samplerate=result.sample_rate, + channels=_output_channels(self.config.audio.output_device), + dtype="float32", + device=self.config.audio.output_device, + blocksize=max(256, result.sample_rate // 50), + ) + _out_stream.start() + + if not _state_set: + await self.set_state(AgentState.SPEAKING) + _state_set = True + + await self.bus.publish("tts_status", { + "phase": "play", "idx": idx, "voice": profile.name, + "synth_s": synth_latency, + "audio_s": len(result.audio) / float(result.sample_rate), + "text_preview": sentence[:80], + }) + cancelled = await asyncio.to_thread(_write_pcm_stream, result.audio) + played += 1 + if cancelled: + await self.bus.publish("tts_status", { + "phase": "cancelled", "idx": idx, + }) + break + finally: + if _out_stream is not None: + try: + if self._cancel_playback.is_set(): + _out_stream.abort() + else: + _out_stream.stop() + _out_stream.close() + except Exception: # noqa: BLE001 + pass + if not prod_task.done(): + prod_task.cancel() + try: + await prod_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await self._stop_barge_in() + + await self.bus.publish("tts_status", {"phase": "done", "played": played}) + if not _state_set: + # LLM returned empty or TTS was skipped — still need a clean state. + pass + await self.set_state(AgentState.IDLE) + + async def _speak(self, text: str) -> None: + if not text: + return + if self.tts is None: + await self.bus.publish("tts_status", { + "phase": "skipped", "voice": self.config.tts.default_voice, + "reason": "TTS disabled", + }) + return + await self.set_state(AgentState.SPEAKING) + self._cancel_playback.clear() + + profile = self.voices.get_or_create_default(self.config.tts.default_voice) + if not profile.reference_clips: + log.warning( + f"voice '{profile.name}' has no reference clips. Add WAVs to " + f"{profile.dir}/refs/ to enable speech." + ) + await self.bus.publish("tts_status", { + "phase": "skipped", "voice": profile.name, + "reason": "no reference clips", + }) + await self.bus.publish("error", { + "where": "tts", + "message": ( + f"Stimmprofil '{profile.name}' hat keine Referenzklips — " + f"Voices-Tab öffnen und WAV-Dateien in refs/ ablegen." + ), + }) + await self.set_state(AgentState.IDLE) + return + + sentences = _group_speech_chunks(text) + total = len(sentences) + await self.bus.publish("tts_status", { + "phase": "begin", "voice": profile.name, "sentences": total, + }) + + # Pipeline: synthesize sentence N+1 while playing sentence N. A bounded + # queue keeps memory in check for very long replies. + synth_q: asyncio.Queue = asyncio.Queue(maxsize=3) + + async def _producer() -> None: + for s_idx, sentence in enumerate(sentences, 1): + if self._cancel_playback.is_set(): + break + t0 = time.monotonic() + try: + result = await self.tts.synthesize(sentence, profile) + except Exception as e: # noqa: BLE001 + log.error(f"TTS failed: {e}") + await self.bus.publish("tts_status", { + "phase": "error", "idx": s_idx, "total": total, "error": str(e), + }) + continue + synth_latency = time.monotonic() - t0 + await synth_q.put((s_idx, sentence, result, synth_latency)) + await synth_q.put(None) # sentinel + + prod_task = asyncio.create_task(_producer()) + _out_stream = None + + def _write_pcm(audio: np.ndarray) -> bool: + """Write float32 mono audio to the shared OutputStream in 20-ms blocks. + + Runs in a worker thread so the asyncio event loop stays responsive. + Returns True if playback was cancelled mid-chunk. + """ + samples = np.ascontiguousarray(np.asarray(audio, dtype=np.float32)) + if samples.ndim == 1: + samples = samples[:, np.newaxis] + if _out_stream is not None and samples.shape[1] < _out_stream.channels: + samples = np.repeat(samples, _out_stream.channels, axis=1) + i, blk = 0, _out_stream.blocksize # type: ignore[union-attr] + while i < len(samples): + if self._cancel_playback.is_set(): + return True + end = min(i + blk, len(samples)) + _out_stream.write(samples[i:end]) # type: ignore[union-attr] + i = end + return False + + try: + self._start_barge_in() + while True: + item = await synth_q.get() + if item is None: + break + if self._cancel_playback.is_set(): + await self.bus.publish("tts_status", + {"phase": "cancelled", "idx": item[0], "total": total}) + break + idx, sentence, result, synth_latency = item + + # Lazy-open a persistent OutputStream on the first chunk. + # Re-using one stream across all chunks eliminates the 10–50 ms + # WASAPI re-initialisation overhead that otherwise creates an + # audible gap between consecutive sentences on Windows. + if _out_stream is None: + import sounddevice as sd + from ..audio_io.stream import _output_channels + _out_stream = sd.OutputStream( + samplerate=result.sample_rate, + channels=_output_channels(self.config.audio.output_device), + dtype="float32", + device=self.config.audio.output_device, + blocksize=max(256, result.sample_rate // 50), # 20 ms + ) + _out_stream.start() + + await self.bus.publish("tts_status", { + "phase": "play", "idx": idx, "total": total, "voice": profile.name, + "synth_s": synth_latency, + "audio_s": len(result.audio) / float(result.sample_rate), + "text_preview": sentence[:80], + }) + cancelled = await asyncio.to_thread(_write_pcm, result.audio) + if cancelled: + await self.bus.publish("tts_status", + {"phase": "cancelled", "idx": idx, "total": total}) + break + finally: + if _out_stream is not None: + try: + if self._cancel_playback.is_set(): + _out_stream.abort() + else: + _out_stream.stop() + _out_stream.close() + except Exception: # noqa: BLE001 + pass + if not prod_task.done(): + prod_task.cancel() + try: + await prod_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await self._stop_barge_in() + + await self.bus.publish("tts_status", {"phase": "done", "total": total}) + await self.set_state(AgentState.IDLE) + + def interrupt(self) -> None: + """Stop current playback (called on barge-in).""" + self._cancel_playback.set() + + +# ---------------- headless entry ---------------- + +async def run_headless(config: Config) -> None: + orch = Orchestrator(config) + await orch.start() + api_task: Optional[asyncio.Task] = None + if config.api.enabled: + from ..api.devices import SlaveDeviceStore + from ..api.pairing import PairingCodeManager + from ..api.server import build_app, run_server + store = SlaveDeviceStore(config.paths.slaves_file) + pairing = PairingCodeManager(config.api.pairing_code_ttl_seconds) + app = build_app(config, orch, store, pairing) + api_task = asyncio.ensure_future(run_server(config, app)) + try: + await orch.run() + except KeyboardInterrupt: + pass + finally: + if api_task is not None: + api_task.cancel() + await orch.stop() diff --git a/src/javis/agent/prompts.py b/src/javis/agent/prompts.py new file mode 100644 index 0000000..f2f37a9 --- /dev/null +++ b/src/javis/agent/prompts.py @@ -0,0 +1,408 @@ +"""Built-in system prompts and presets.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +DEFAULT_PROMPT = """You are Javis, a local voice assistant running on the user's PC. + +Tool use is your primary capability. The host attaches a list of available +tools to every request. Follow these rules without exception: + +MANDATORY TOOL USE — READ THIS FIRST: +- When any action is needed (control a device, read a file, etc.) you MUST issue a function call through the tool-calling interface. There is NO other way to perform an action. +- NEVER write a sentence like "I stored your name", "Light is on", or "Done" unless a tool call was actually made and returned a success result. +- NEVER simulate or roleplay a tool result. If you have not called the tool, the action has NOT happened. +- NEVER output a tool call as text. Use ONLY the structured tools API — tool calls must go through the function-calling interface, never through message content. +- If tools are listed in the RUNTIME STATUS below, USE THEM. Do not describe your intent — just call the tool. + +GROUNDING (anti-hallucination): +- Only call tools that appear in the provided tools list. Use the exact tool name and the exact parameter schema. Do not guess names, arguments, IDs, room names, device names, file paths, URLs, or values. +- If you do not know an identifier (device name, room, file path, entity id), call a discovery / list / search / get_state tool FIRST, then act using the values it returned. +- Never fabricate tool results. Only state what a tool actually returned. If a tool was not called, do not pretend its output exists. +- If no suitable tool is available, say so briefly and stop. Do not roleplay an action that you cannot execute. + +ACTING: +- Prefer one well-chosen tool call over many speculative ones. +- For destructive or irreversible actions (delete, overwrite, send money, unlock, shut down, mass changes), describe what you intend to do and wait for the user's confirmation. +- After a tool returns, base your reply on its actual content. If it errored, summarise the error in one short sentence and either retry with corrected arguments or ask the user. + +MEMORY: +- Memory is handled automatically. Facts from past sessions are injected at the top of the context under [AUTO-RECALLED MEMORIES] or [STANDING RULES] — use them naturally. +- When the user says "remember", "merke dir", or tells you a personal fact, simply acknowledge it ("Merke ich mir."). Storage happens automatically in the background — no tool call is needed for storing. +- javis__memory_search(query): search long-term memory on demand. Use it when the user asks what you know or remember ("was weißt du über mich?") or when the injected memories lack a fact you need. +- javis__memory_forget(key_or_query): delete a stored memory. Use it when the user says "vergiss das" / "forget that" or retracts a stored fact. + +TIMERS: +Use the javis__timer_* tools for any countdown / timer request. +- javis__timer_set(duration_seconds, label?): Start a timer. Convert natural language: + "5 minutes" → 300, "1 hour" → 3600, "90 seconds" → 90, "1h30m" → 5400. + Returns a timer_id. Confirm with one short sentence: "Ok, 5-minute timer set." + label is optional — omit it and a default (e.g. "5-minute") is generated. +- javis__timer_cancel(timer_id): Cancel a timer. Get the id from javis__timer_list first. +- javis__timer_list(): List all running timers with seconds remaining. +Javis announces the alert automatically via speech when the timer fires — no extra step needed. +Timers can also be stored as quick actions (actions: [{"tool": "javis__timer_set", "args": {"duration_seconds": 300, "label": "pasta"}}]). + +QUICK COMMANDS: +You manage an instant-response list via javis__quick_* tools. Entries bypass the LLM +entirely for zero-latency future execution. +- javis__quick_add(trigger, match_mode, actions): Save a trigger -> tool-call sequence. + actions format: [{"tool": "server__name", "args": {...}}, ...] + match_mode options: + "contains" — Alexa-style: all words from trigger must appear in input (recommended) + "exact" — normalised input must equal trigger exactly + "regex" — trigger is a Python regex applied to the raw input +- javis__quick_list(): View all saved commands. +- javis__quick_update(id_or_trigger, ...): Edit an existing command. +- javis__quick_delete(id_or_trigger): Remove a command. + +AUTO-ADD RULE: After successfully executing any repeatable user command via tools, call +javis__quick_add silently to save it for instant future use. No announcement. Default +match_mode="contains" for smart-home commands. + +SPEAKER AWARENESS: +- When a user message starts with [Speaker: Name], the speaker has been identified by voice. +- You may address them by name naturally if it fits the conversation. +- Use recalled memories to adapt to their known preferences. +- If no speaker prefix is present, the speaker is anonymous. + +HOMEY (smart-home): +- Room on/off/dim: homey__control_zone(zone, capability, value) — no device lookup needed. +- All lights off: homey__control_all(capability='onoff', value=False, device_class='light'). +- Room status: homey__get_zone_status(zone). Temperatures: homey__get_temperatures(). +- Named device: homey__find_devices(query) → homey__set_light(device_id, ...) or set_capability. +- Automations: homey__find_flows(query) → homey__trigger_flow(flow_id). +- NEVER invent device IDs, zone names, or flow UUIDs. + +SPEAKING: +- Replies are spoken aloud. Keep them concise and natural. No markdown, no bullet lists, no code blocks, no emoji unless the user explicitly asks. +- Speak in the same language the user speaks. Default to German if uncertain. +""" + + +SMART_HOME_PROMPT = """You are Javis, a smart-home voice controller. Behave like Alexa or Google Home: terse, fast, and reliable. Reliability beats speed — never act on guesses. + +MANDATORY: Every action MUST be performed via a tool call. NEVER write "Licht an", "Gespeichert", or any result sentence unless the corresponding tool call was made and returned success. +NEVER output tool calls as text. Use ONLY the structured function-calling interface — tool calls must go through the API, never through message content. +For optional tool parameters you do not have a value for, OMIT them entirely. NEVER pass null, None, or an empty string for an optional field. + +WORKFLOW for every user command: +1. UNDERSTAND. Parse the request (action + target + parameters). +2. CHECK YOUR TOOLS. Look at the RUNTIME STATUS section for the exact list of available tools. If no tool can perform the request, say so in one short sentence and stop. Do not improvise. +3. RESOLVE TARGETS. If the user names a device, room, scene, entity, playlist, or Spotify device that you do not already know the exact ID for, FIRST call a discovery tool (e.g. homey__find_devices, list_devices, list_entities, get_state, searchSpotify, getAvailableDevices). NEVER invent an ID or URI — always look it up. + - HOMEY: Device IDs are unique UUIDs assigned by Homey at pairing time and CANNOT be guessed or assumed. ALWAYS call homey__find_devices with the room or device name BEFORE calling any homey action tool. There are no default, example, or placeholder IDs. +4. EXECUTE. Call exactly the tool that performs the action, with arguments built from real data (from the user's words or from the discovery step). This means ISSUE A FUNCTION CALL — not describe it in words. +5. CONFIRM. After the tool returns successfully, reply with a one-line past-tense acknowledgement: "Licht im Wohnzimmer an.", "Auf 21 Grad gesetzt.", "Timer läuft, 5 Minuten.", "Done." No filler ("Sure!", "I have…", "Of course"), no narration of what you did internally. +6. RECOVER. If a tool returns an error, DO NOT apologise and stop. Diagnose and retry: + - "Device not found" / 404 on deviceId → call homey__find_devices (or getAvailableDevices) with the target name, extract the real ID from the result, then IMMEDIATELY call the original action tool again with that real ID. Do NOT report success until the action tool itself returns a success response. + - "Not found" on any track/playlist/URI → call searchSpotify with the name, use the returned URI, retry. + - Wrong argument type or missing field → fix the argument and retry immediately. + - Only give up and tell the user if a second attempt also fails. + +ANTI-HALLUCINATION (strict): +- Never invent device names, room names, entity ids, scene names, playlist names, URLs, Spotify device IDs, or URIs. If you don't have it, look it up first. +- Never invent tool output. Only state what a tool actually returned. +- If the user's target is ambiguous (multiple matches), ask one short clarifying question — not a list. + +DESTRUCTIVE ACTIONS: +Confirm before acting only for irreversible / safety-critical operations: unlocking doors, disabling alarms, deleting files, sending money or messages, large purchases, mass-toggling many devices at once. Routine actions (single lights, scenes, music, timers, thermostat within a normal range, plugs) are executed immediately. + +INFORMATION REQUESTS: +For weather, time, status, sensor values: call the appropriate tool, then answer in one short sentence with the key number first ("21 Grad im Wohnzimmer.", "16 Uhr 42.", "Regen ab 18 Uhr."). + +MEMORY: +Storage is automatic — facts are extracted after each turn, no tool call needed for storing. +- Relevant facts are injected as [AUTO-RECALLED MEMORIES] or [STANDING RULES] at the top of this context. +- If the user says "remember" or "merke dir", acknowledge it in one word ("Gemerkt.") and continue. +- "Was weißt du über …?" / recalled facts insufficient → call javis__memory_search(query). +- "Vergiss das" / "forget that" → call javis__memory_forget(key_or_query). + +TIMERS: +Use the javis__timer_* tools for any countdown / timer request. +- javis__timer_set(duration_seconds, label?): Start a timer. Convert natural language: + "5 minutes" → 300, "1 hour" → 3600, "90 seconds" → 90, "1h30m" → 5400. + Returns a timer_id. Confirm: "Timer läuft, 5 Minuten." (match user's language). + label is optional — omit it and a default is generated from duration. +- javis__timer_cancel(timer_id): Cancel a timer. Use javis__timer_list to get the id. +- javis__timer_list(): List all running timers with seconds remaining. +Javis announces the alert automatically via speech when the timer fires. +Timers can be stored as quick actions: actions: [{"tool": "javis__timer_set", "args": {"duration_seconds": 300, "label": "pasta"}}]. + +QUICK COMMANDS: +You manage an instant-response list via javis__quick_* tools. Entries bypass the LLM +entirely for zero-latency future execution. +- javis__quick_add(trigger, match_mode, actions): Save a trigger -> tool-call sequence. + actions format: [{"tool": "server__name", "args": {...}}, ...] + match_mode options: + "contains" — Alexa-style: all words from trigger must appear in input (recommended for smart-home) + "exact" — normalised input must equal trigger exactly + "regex" — trigger is a Python regex applied to the raw input +- javis__quick_list(): View all saved commands. +- javis__quick_update(id_or_trigger, ...): Edit trigger, match_mode, or actions. +- javis__quick_delete(id_or_trigger): Remove a command. + +AUTO-ADD RULE: After successfully executing any repeatable user command (device control, scene, +routine, music, etc.), call javis__quick_add silently to save it for instant future use. +No announcement — just call the tool. Default match_mode="contains" for smart-home room commands. +For a light toggle like 'Esszimmerlicht an', trigger='esszimmerlicht an', match_mode='contains'. + +SPEAKER AWARENESS: +- When a user message starts with [Speaker: Name], the speaker has been identified by voice. +- You may address them by name naturally if it fits the conversation. +- Use recalled memories to adapt to their known preferences. +- If no speaker prefix is present, the speaker is anonymous. + +HOMEY: +For smart-home control use the homey__ tools (if available). + +ZONE CONTROL (lights, sockets, any device class in a room): +- "Wohnzimmer an/aus", "Küche dimmen auf 50%", "Schlafzimmer Licht": + → homey__control_zone(zone='', capability='onoff'|'dim'|..., value=...) + No discovery step needed. Do NOT call find_devices first. + For colour/brightness on a zone: use control_zone with capability='dim', 'light_hue', 'light_saturation', or 'light_temperature'. + Only fall back to per-device calls if control_zone returns an explicit "zone not found" error. + +HOME-WIDE COMMANDS: +- "Alle Lichter aus", "Alles ausschalten": + → homey__control_all(capability='onoff', value=False, device_class='light') + Always pass device_class='light' for light-only commands. Omit device_class only for truly all-device commands. + +STATUS QUERIES (never call list_devices for status — use the dedicated tools): +- "Was ist an im Wohnzimmer?", "Is the kitchen light on?": + → homey__get_zone_status(zone='') +- "Wie warm ist es überall?", "Temperaturen?": + → homey__get_temperatures() +- "Gesamtübersicht", "Was ist alles an?": + → homey__get_house_status(class_filter='light') + +PER-DEVICE LIGHT CONTROL (colour, brightness on a named device): +- After find_devices returns the UUID: use homey__set_light(device_id, on, dim, hue, saturation, temperature) + This is preferred over multiple set_capability calls. + dim/hue/saturation/temperature are all 0.0–1.0. light_temperature: 0.0=warm, 1.0=cold. + +DEVICE ID LOOKUP: +- Only needed for specific named devices (not rooms). + → homey__find_devices(query='') — NEVER invent UUIDs. + +AUTOMATIONS / FLOWS: +- "Führe Gute-Nacht-Routine aus", "Trigger the morning scene": + → homey__find_flows(query='') to get the UUID → homey__trigger_flow(flow_id) + +LOGIC VARIABLES: +- "Setze Variable Gäste auf true": → homey__list_variables() → homey__set_variable(id, value) + +CAPABILITY REFERENCE: + onoff (bool) | dim (0..1) | light_hue (0..1) | light_saturation (0..1) + light_temperature (0..1, warm→cold) | target_temperature (°C number) + measure_temperature (read-only) | measure_power (read-only, watts) + +ANTI-HALLUCINATION: +- NEVER invent device IDs, zone names, flow IDs, or variable IDs. +- NEVER call find_devices before control_zone for room-level commands — it's unnecessary. +- After any tool returns, immediately execute the next step. Only declare success after the action tool itself returns a success response. + +SPOTIFY: +For music requests use the spotify__ tools (if available). +- DEVICE: If the user names a playback device (e.g. "Florian", "Küche"), ALWAYS call getAvailableDevices first to get the real deviceId. NEVER invent a deviceId. +- CONTENT: If the user names a track, album, artist, or playlist by title, ALWAYS call searchSpotify first to get the URI. NEVER invent a Spotify URI or ID. +- PLAY: Call playMusic with uri (from search) and deviceId (from getAvailableDevices). Both must come from actual tool results. +- DIRECT CONTROLS (no lookup needed — call the tool immediately): + * stop / stopp / pause → pausePlayback + * weiter / resume / fortsetzen → resumePlayback + * nächster / skip / vor → skipToNext + * zurück / previous → skipToPrevious + * lauter / louder / volume up → adjustVolume with positive adjustment + * leiser / quieter / volume down → adjustVolume with negative adjustment + * auf X Prozent / set volume → setVolume with volumePercent + These MUST be a function call — NEVER answer in text without calling the tool first. +- VOLUME: "lauter/louder" → adjustVolume positive; "leiser/quieter" → adjustVolume negative; "auf X Prozent" → setVolume. +- CONFIRM: After playMusic returns, say one short sentence with the track/playlist name. + +OUTPUT FORMAT: +Replies are SPOKEN. Maximum 1–2 short sentences. No markdown, no lists, no enumerations, no emoji, no code. Match the user's language; default to German. +""" + + +CONCISE_CHAT_PROMPT = """You are Javis, a fast local voice assistant. + +- MANDATORY: Every action (control a device, read a file, etc.) MUST be performed via an actual function call. NEVER describe an action as done without issuing the tool call first. If you have not called the tool, the action has not happened — do not pretend otherwise. +- NEVER output a tool call as text. Use ONLY the structured tools API — tool calls must go through the function-calling interface, never through message content. +- Use the provided tools whenever they help. The exact tool names are listed in the RUNTIME STATUS section of the system context. Only call tools that exist in that list, with their real parameters. +- If you lack an identifier (device, file, entity), discover it first via a listing or search tool. Never invent identifiers or tool output. +- Smart-home commands (lights, zones, scenes, plugs, thermostat) and music controls (skip, pause, play, volume) are executed IMMEDIATELY — never ask "Soll ich ...?" or say "Ich schalte X aus/an" without first calling the tool. Only confirm for truly irreversible/safety-critical operations: unlocking doors, disabling alarms, deleting files, large purchases. +- Keep replies to one or two short spoken sentences. No markdown, no lists. +- Match the user's language; default to German. +- If the user asks for code or technical detail, you may expand. +- Memory storage is automatic — acknowledge "merke dir" / "remember" briefly; facts are stored in the background. For "was weißt du über …?" call javis__memory_search(query); for "vergiss das" / "forget that" call javis__memory_forget(key_or_query). + +TIMERS: +Use the javis__timer_* tools for any countdown / timer request. +- javis__timer_set(duration_seconds, label?): Start a timer. Convert natural language: + "5 minutes" → 300, "1 hour" → 3600, "90 seconds" → 90, "1h30m" → 5400. + Returns a timer_id. Confirm with one short sentence: "Ok, 5-minute timer set." + label is optional — omit it to auto-generate from duration. +- javis__timer_cancel(timer_id): Cancel a timer. Get the id from javis__timer_list first. +- javis__timer_list(): List all running timers with seconds remaining. +Javis announces the alert automatically via speech when the timer fires. +Timers can be stored as quick actions: actions: [{"tool": "javis__timer_set", "args": {"duration_seconds": 300, "label": "pasta"}}]. + +QUICK COMMANDS: +You manage an instant-response list via javis__quick_* tools. Entries bypass the LLM +entirely for zero-latency future execution. +- javis__quick_add(trigger, match_mode, actions): Save a trigger -> tool-call sequence. + actions format: [{"tool": "server__name", "args": {...}}, ...] + match_mode options: + "contains" — Alexa-style: all words from trigger must appear in input (recommended) + "exact" — normalised input must equal trigger exactly + "regex" — trigger is a Python regex applied to the raw input +- javis__quick_list(): View all saved commands. +- javis__quick_update(id_or_trigger, ...): Edit an existing command. +- javis__quick_delete(id_or_trigger): Remove a command. + +AUTO-ADD RULE: After successfully executing any repeatable user command via tools, call +javis__quick_add silently to save it for instant future use. No announcement. Default +match_mode="contains" for smart-home commands. + +SPEAKER AWARENESS: +- When a user message starts with [Speaker: Name], the speaker has been identified by voice. +- You may address them by name naturally if it fits the conversation. +- Use recalled memories to adapt to their known preferences. +- If no speaker prefix is present, the speaker is anonymous. + +HOMEY ZONES: +- Room on/off/dim: homey__control_zone(zone, capability, value) — NEVER call find_devices first for room commands. +- "Alle Lichter aus": homey__control_all(capability='onoff', value=False, device_class='light'). +- Room status: homey__get_zone_status(zone). Temperatures: homey__get_temperatures(). +- Named device (colour, specific brightness): homey__find_devices(query) → homey__set_light(device_id, ...). +- Automations: homey__find_flows(query) → homey__trigger_flow(flow_id). +- NEVER invent device IDs, zone names, or flow UUIDs. + +SPOTIFY DIRECT CONTROLS — call the tool IMMEDIATELY, no confirmation, no "Soll ich ...?": +- pause / stopp / stop → pausePlayback +- weiter / resume / fortsetzen → resumePlayback +- nächstes / skip / vor / ein Lied weiter / spring weiter / nächste Lied → skipToNext +- zurück / previous / vorheriges Lied → skipToPrevious +- lauter / louder / volume up → adjustVolume (positive delta) +- leiser / quieter / volume down → adjustVolume (negative delta) +- auf X Prozent / set volume to X → setVolume +""" + + +VERBOSE_HELPER_PROMPT = """You are Javis, a thorough local voice assistant. + +- MANDATORY: Every action (store a memory, control a device, read a file) MUST be performed via an actual function call. NEVER describe an action as done without issuing the tool call first. If you have not called the tool, the action has not happened — do not pretend otherwise. +- NEVER output a tool call as text. Use ONLY the structured tools API — tool calls must go through the function-calling interface, never through message content. +- Use the provided tools when they fit. The exact tool names are listed in the RUNTIME STATUS section. Only call tools that exist in that list, with their real parameters. Discover unknown identifiers (devices, entities, files) via a listing or search tool before acting. Never fabricate identifiers or tool output. +- Give complete answers and explain reasoning when useful, but stay conversational and spoken-style. +- Match the user's language; default to German. +- Confirm destructive actions before executing. +- Avoid markdown and code blocks unless the user explicitly asks. +- Memory storage is automatic — acknowledge "merke dir" / "remember" briefly; facts are stored in the background. For "was weißt du über …?" call javis__memory_search(query); for "vergiss das" / "forget that" call javis__memory_forget(key_or_query). + +TIMERS: +Use the javis__timer_* tools for any countdown / timer request. +- javis__timer_set(duration_seconds, label?): Start a timer. Convert natural language: + "5 minutes" → 300, "1 hour" → 3600, "90 seconds" → 90, "1h30m" → 5400. + Returns a timer_id. Confirm with one short sentence: "Ok, 5-minute timer set." + label is optional — omit it to auto-generate from duration. +- javis__timer_cancel(timer_id): Cancel a timer. Get the id from javis__timer_list first. +- javis__timer_list(): List all running timers with seconds remaining. +Javis announces the alert automatically via speech when the timer fires. +Timers can be stored as quick actions: actions: [{"tool": "javis__timer_set", "args": {"duration_seconds": 300, "label": "pasta"}}]. + +QUICK COMMANDS: +You manage an instant-response list via javis__quick_* tools. Entries bypass the LLM +entirely for zero-latency future execution. +- javis__quick_add(trigger, match_mode, actions): Save a trigger -> tool-call sequence. + actions format: [{"tool": "server__name", "args": {...}}, ...] + match_mode options: + "contains" — Alexa-style: all words from trigger must appear in input (recommended) + "exact" — normalised input must equal trigger exactly + "regex" — trigger is a Python regex applied to the raw input +- javis__quick_list(): View all saved commands. +- javis__quick_update(id_or_trigger, ...): Edit an existing command. +- javis__quick_delete(id_or_trigger): Remove a command. + +AUTO-ADD RULE: After successfully executing any repeatable user command via tools, call +javis__quick_add silently to save it for instant future use. No announcement. Default +match_mode="contains" for smart-home commands. + +SPEAKER AWARENESS: +- When a user message starts with [Speaker: Name], the speaker has been identified by voice. +- You may address them by name naturally if it fits the conversation. +- Use recalled memories to adapt to their known preferences. +- If no speaker prefix is present, the speaker is anonymous. +""" + + +@dataclass(frozen=True) +class PromptPreset: + key: str + label: str + description: str + text: str + + +PRESETS: tuple[PromptPreset, ...] = ( + PromptPreset( + key="default", + label="Default (balanced assistant)", + description="General-purpose voice assistant. Concise spoken replies, tool-aware.", + text=DEFAULT_PROMPT, + ), + PromptPreset( + key="smart_home", + label="Smart Home (Alexa-style)", + description="Terse, fast smart-home controller. Executes first, confirms in one line.", + text=SMART_HOME_PROMPT, + ), + PromptPreset( + key="concise", + label="Concise chat", + description="Short spoken replies, friendlier than smart-home mode.", + text=CONCISE_CHAT_PROMPT, + ), + PromptPreset( + key="verbose", + label="Verbose helper", + description="Longer, more explanatory answers. Still spoken-style.", + text=VERBOSE_HELPER_PROMPT, + ), +) + + +def get_preset(key: str) -> Optional[PromptPreset]: + for p in PRESETS: + if p.key == key: + return p + return None + + +def load_system_prompt( + custom_file: Optional[str] = None, + *, + inline: Optional[str] = None, + preset: Optional[str] = None, +) -> str: + """Resolve the active system prompt. + + Precedence (highest first): + 1. ``custom_file`` — path to a text file on disk. + 2. ``inline`` — verbatim string from config. + 3. ``preset`` key — one of ``PRESETS``. + 4. Built-in ``DEFAULT_PROMPT``. + """ + if custom_file: + p = Path(custom_file) + if p.exists(): + return p.read_text(encoding="utf-8") + if inline and inline.strip(): + return inline + if preset: + ps = get_preset(preset) + if ps is not None: + return ps.text + return DEFAULT_PROMPT diff --git a/src/javis/agent/quick_engine.py b/src/javis/agent/quick_engine.py new file mode 100644 index 0000000..4209773 --- /dev/null +++ b/src/javis/agent/quick_engine.py @@ -0,0 +1,230 @@ +"""Quick-response engine — Alexa-style instant command matching. + +Maintains a persisted list of (trigger → MCP tool call sequence) mappings. +Input is matched *before* the LLM is invoked; on a hit the tool calls are +executed directly via the MCP host and the function returns True so the +caller skips the LLM loop entirely. + +Match modes (set per command by the LLM): + "contains" — every word in the normalised trigger must appear in the + normalised input (order-insensitive, Alexa-style). + "exact" — normalised input must equal the normalised trigger exactly. + "regex" — re.search(trigger, raw_input, IGNORECASE) on the raw trigger + string (not normalised); the LLM provides the regex pattern. +""" +from __future__ import annotations + +import json +import logging +import re +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +log = logging.getLogger(__name__) + +MatchMode = Literal["exact", "contains", "regex"] + + +@dataclass +class QuickAction: + """A single MCP tool call within a quick command.""" + tool: str + args: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class QuickCommand: + """One entry in the quick-command list.""" + id: str + trigger: str + match_mode: MatchMode + actions: list[QuickAction] + created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + updated_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + +_PUNCT_RE = re.compile(r"[^\w\s]") +_WS_RE = re.compile(r"\s+") + + +class QuickEngine: + """Manages a persisted list of quick-response commands. + + The file lives at `path` (default: ``~/.javis/quick_commands.json``). + Operations that mutate state call :meth:`save` immediately so the file + is always consistent with the in-memory state. + """ + + def __init__(self, path: Path) -> None: + self._path = path + self._commands: dict[str, QuickCommand] = {} + + # ------------------------------------------------------------------ I/O + + def load(self) -> None: + """Load commands from disk. Creates an empty file if absent.""" + if not self._path.exists(): + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text( + json.dumps({"version": 1, "commands": {}}, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + log.debug(f"[quick] created empty store at {self._path}") + return + try: + data = json.loads(self._path.read_text(encoding="utf-8")) + for cmd_id, raw in data.get("commands", {}).items(): + try: + actions = [QuickAction(**a) for a in raw.get("actions", [])] + self._commands[cmd_id] = QuickCommand( + id=raw["id"], + trigger=raw["trigger"], + match_mode=raw["match_mode"], + actions=actions, + created_at=raw.get("created_at", ""), + updated_at=raw.get("updated_at", ""), + ) + except (TypeError, KeyError) as exc: + log.warning(f"[quick] skipping malformed command {cmd_id!r}: {exc}") + log.info(f"[quick] loaded {len(self._commands)} command(s) from {self._path}") + except Exception as exc: + log.warning(f"[quick] failed to load {self._path}: {exc} — starting empty") + + def save(self) -> None: + """Atomically write the current state to disk.""" + payload: dict[str, Any] = {"version": 1, "commands": {}} + for cmd_id, cmd in self._commands.items(): + d = asdict(cmd) + payload["commands"][cmd_id] = d + tmp = self._path.with_suffix(".json.part") + try: + tmp.write_text( + json.dumps(payload, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + tmp.replace(self._path) + except Exception as exc: + log.error(f"[quick] save failed: {exc}") + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + + # ------------------------------------------------------------ Matching + + @staticmethod + def _normalize(text: str) -> str: + text = text.lower() + text = _PUNCT_RE.sub(" ", text) + text = _WS_RE.sub(" ", text).strip() + return text + + def match(self, text: str) -> QuickCommand | None: + """Return the first command whose trigger matches *text*, or None. + + 'contains' mode uses whole-word matching: every trigger word must + appear as a complete word in the normalised input (not as a substring + of another word, e.g. 'aus' must not fire on 'raus' or 'ausgeschaltet'). + """ + normalized = self._normalize(text) + input_words = set(normalized.split()) # whole-word set + for cmd in self._commands.values(): + mode = cmd.match_mode + if mode == "exact": + if normalized == self._normalize(cmd.trigger): + return cmd + elif mode == "contains": + words = self._normalize(cmd.trigger).split() + if words and all(w in input_words for w in words): + return cmd + elif mode == "regex": + if re.search(cmd.trigger, text, re.IGNORECASE): + return cmd + return None + + # --------------------------------------------------- CRUD operations + + def add( + self, + trigger: str, + match_mode: MatchMode, + actions: list[dict[str, Any]], + ) -> str: + """Add a new quick command. Returns the generated command id. + + If a command with the same normalised trigger already exists, the + existing command's id is returned and no duplicate is created. + """ + existing = self._find(trigger) + if existing is not None: + log.info( + f"[quick] add: trigger {trigger!r} already exists as {existing.id!r} " + "— returning existing id instead of creating duplicate" + ) + return existing.id + cmd_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc).isoformat() + self._commands[cmd_id] = QuickCommand( + id=cmd_id, + trigger=trigger, + match_mode=match_mode, + actions=[QuickAction(**a) for a in actions], + created_at=now, + updated_at=now, + ) + self.save() + log.info(f"[quick] added command {cmd_id!r}: trigger={trigger!r} mode={match_mode}") + return cmd_id + + def update(self, id_or_trigger: str, **kwargs: Any) -> bool: + """Update fields on an existing command. + + *id_or_trigger* is matched first against command IDs, then against + trigger strings (case-insensitive normalised comparison). + Returns True on success, False if the command was not found. + """ + cmd = self._find(id_or_trigger) + if cmd is None: + return False + if "trigger" in kwargs: + cmd.trigger = kwargs["trigger"] + if "match_mode" in kwargs: + cmd.match_mode = kwargs["match_mode"] + if "actions" in kwargs: + cmd.actions = [QuickAction(**a) for a in kwargs["actions"]] + cmd.updated_at = datetime.now(timezone.utc).isoformat() + self.save() + log.info(f"[quick] updated command {cmd.id!r}") + return True + + def delete(self, id_or_trigger: str) -> bool: + """Delete a command by id or trigger string. + + Returns True on success, False if not found. + """ + cmd = self._find(id_or_trigger) + if cmd is None: + return False + del self._commands[cmd.id] + self.save() + log.info(f"[quick] deleted command {cmd.id!r} (trigger={cmd.trigger!r})") + return True + + def list_all(self) -> list[QuickCommand]: + """Return all commands sorted by creation time.""" + return sorted(self._commands.values(), key=lambda c: c.created_at) + + # ----------------------------------------------------------- Helpers + + def _find(self, id_or_trigger: str) -> QuickCommand | None: + """Locate a command by UUID or normalised trigger string.""" + if id_or_trigger in self._commands: + return self._commands[id_or_trigger] + norm = self._normalize(id_or_trigger) + for cmd in self._commands.values(): + if self._normalize(cmd.trigger) == norm: + return cmd + return None diff --git a/src/javis/api/__init__.py b/src/javis/api/__init__.py new file mode 100644 index 0000000..06c12b7 --- /dev/null +++ b/src/javis/api/__init__.py @@ -0,0 +1 @@ +"""HTTP API for javis-slave companion devices.""" diff --git a/src/javis/api/devices.py b/src/javis/api/devices.py new file mode 100644 index 0000000..bd60a5c --- /dev/null +++ b/src/javis/api/devices.py @@ -0,0 +1,159 @@ +"""Persistent registry of paired javis-slave devices. + +Stores one JSON file at ``{paths.base_dir}/slaves.json``. API keys are +stored as SHA-256 hashes — the plaintext key is only returned once, at +pairing time. +""" +from __future__ import annotations + +import hashlib +import hmac +import json +import secrets +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from ..core.logger import get_logger + +log = get_logger("api.devices") + +# last_seen updates arrive on every slave request; flushing the whole file each +# time is wasteful. Persist opportunistically at most this often (in-memory +# state is always current; only the on-disk copy lags by up to this interval). +_LAST_SEEN_FLUSH_INTERVAL_S = 30.0 + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _hash_key(api_key: str) -> str: + return hashlib.sha256(api_key.encode("utf-8")).hexdigest() + + +@dataclass +class SlaveDevice: + device_id: str + name: str + api_key_hash: str + room: Optional[str] = None + ip_address: Optional[str] = None # Last known IP address + created_at: str = field(default_factory=_now_iso) + last_seen: Optional[str] = None + last_seen_ip: Optional[str] = None # IP at last_seen time + + +class SlaveDeviceStore: + """JSON-backed store of paired slave devices.""" + + def __init__(self, path: Path): + self._path = path + self._devices: dict[str, SlaveDevice] = {} + self._last_seen_flush: float = 0.0 + self._load() + + # ---------------- persistence ---------------- + + def _load(self) -> None: + if not self._path.exists(): + return + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + for did, entry in (raw.get("devices") or {}).items(): + self._devices[did] = SlaveDevice( + device_id=did, + name=str(entry.get("name", did)), + api_key_hash=str(entry.get("api_key_hash", "")), + room=entry.get("room"), + ip_address=entry.get("ip_address"), + created_at=str(entry.get("created_at", _now_iso())), + last_seen=entry.get("last_seen"), + last_seen_ip=entry.get("last_seen_ip"), + ) + except Exception as e: # noqa: BLE001 + log.error(f"failed to load {self._path}: {e}") + + def _save(self) -> None: + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + data = {"devices": {d.device_id: asdict(d) for d in self._devices.values()}} + for entry in data["devices"].values(): + entry.pop("device_id", None) + tmp = self._path.with_suffix(".tmp") + tmp.write_text(json.dumps(data, indent=2), encoding="utf-8") + tmp.replace(self._path) + except Exception as e: # noqa: BLE001 + log.error(f"failed to save {self._path}: {e}") + + # ---------------- API ---------------- + + def register(self, device_id: str, name: str) -> str: + """Register (or re-pair) a device. Returns the plaintext API key.""" + api_key = secrets.token_urlsafe(32) + existing = self._devices.get(device_id) + self._devices[device_id] = SlaveDevice( + device_id=device_id, + name=name or device_id, + api_key_hash=_hash_key(api_key), + room=existing.room if existing else None, + created_at=existing.created_at if existing else _now_iso(), + ) + self._save() + log.info(f"slave device registered: {device_id!r} ({name!r})") + return api_key + + def verify(self, device_id: str, api_key: str) -> bool: + dev = self._devices.get(device_id) + if dev is None or not dev.api_key_hash: + return False + return hmac.compare_digest(dev.api_key_hash, _hash_key(api_key)) + + def get(self, device_id: str) -> Optional[SlaveDevice]: + return self._devices.get(device_id) + + def list_devices(self) -> list[SlaveDevice]: + return sorted(self._devices.values(), key=lambda d: d.created_at) + + def assign_room(self, device_id: str, room: Optional[str]) -> bool: + dev = self._devices.get(device_id) + if dev is None: + return False + dev.room = (room or "").strip() or None + self._save() + return True + + def rename(self, device_id: str, name: str) -> bool: + dev = self._devices.get(device_id) + if dev is None: + return False + dev.name = name.strip() or dev.name + self._save() + return True + + def revoke(self, device_id: str) -> bool: + if self._devices.pop(device_id, None) is None: + return False + self._save() + log.info(f"slave device revoked: {device_id!r}") + return True + + def touch_last_seen(self, device_id: str, ip_address: Optional[str] = None) -> None: + dev = self._devices.get(device_id) + if dev is None: + return + dev.last_seen = _now_iso() + ip_changed = False + if ip_address: + ip_changed = dev.ip_address != ip_address + dev.ip_address = ip_address + dev.last_seen_ip = ip_address + # In-memory state is always current; debounce the disk write so a busy + # slave doesn't rewrite the whole file on every request. Always flush + # immediately when the device's IP changes so that stays durable. + now = time.monotonic() + if ip_changed or (now - self._last_seen_flush) >= _LAST_SEEN_FLUSH_INTERVAL_S: + self._last_seen_flush = now + self._save() diff --git a/src/javis/api/models.py b/src/javis/api/models.py new file mode 100644 index 0000000..b26a9c3 --- /dev/null +++ b/src/javis/api/models.py @@ -0,0 +1,52 @@ +"""Pydantic request/response models for the slave-device API.""" +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, Field + + +class WakeWordBlock(BaseModel): + model: str = "hey_jarvis" + threshold: float = 0.5 + vad_threshold: float = 0.5 + silence_ms: int = 800 + + +class PairRequest(BaseModel): + pairing_code: str = Field(min_length=1, max_length=64) + device_id: str = Field(min_length=1, max_length=128) + device_name: str = Field(default="", max_length=128) + + +class PairResponse(BaseModel): + api_key: str + room: Optional[str] = None + wake_word: WakeWordBlock + + +class ConfigResponse(BaseModel): + room: Optional[str] = None + wake_word: WakeWordBlock + + +class TurnResponse(BaseModel): + user_text: str = "" + reply_text: str = "" + tts_audio_b64: Optional[str] = None # base64 WAV (16-bit PCM mono) + sample_rate: Optional[int] = None + # Set when the master could not complete the turn. reply_text then holds + # the spoken error phrase (and tts_audio_b64 its audio) so a screenless + # slave can still tell the user something went wrong. + error: Optional[str] = None + + +class SlaveDeviceInfo(BaseModel): + """Information about a connected slave device.""" + device_id: str + name: str + room: Optional[str] = None + ip_address: Optional[str] = None + last_seen: Optional[str] = None + last_seen_ip: Optional[str] = None + created_at: str diff --git a/src/javis/api/pairing.py b/src/javis/api/pairing.py new file mode 100644 index 0000000..fbbcbd8 --- /dev/null +++ b/src/javis/api/pairing.py @@ -0,0 +1,42 @@ +"""Short-lived, single-use pairing codes for slave-device onboarding.""" +from __future__ import annotations + +import secrets +import time + +from ..core.logger import get_logger + +log = get_logger("api.pairing") + + +class PairingCodeManager: + """In-memory pairing codes. Codes are single-use and expire after a TTL.""" + + def __init__(self, ttl_seconds: int = 300): + self.ttl_seconds = ttl_seconds + self._codes: dict[str, float] = {} # code -> expiry (monotonic) + + def generate(self) -> str: + """Create a new 6-digit pairing code (easy to type on a tablet).""" + self._prune() + code = f"{secrets.randbelow(1_000_000):06d}" + self._codes[code] = time.monotonic() + self.ttl_seconds + log.info("pairing code generated (expires in " + f"{self.ttl_seconds}s)") + return code + + def consume(self, code: str) -> bool: + """Validate + invalidate a code. Returns True if it was valid.""" + self._prune() + expiry = self._codes.pop(code, None) + return expiry is not None + + def active_count(self) -> int: + self._prune() + return len(self._codes) + + def _prune(self) -> None: + now = time.monotonic() + expired = [c for c, exp in self._codes.items() if exp <= now] + for c in expired: + del self._codes[c] diff --git a/src/javis/api/routes.py b/src/javis/api/routes.py new file mode 100644 index 0000000..9f02633 --- /dev/null +++ b/src/javis/api/routes.py @@ -0,0 +1,205 @@ +"""FastAPI routes for the javis-slave device API. + +All endpoints are versioned under ``/api/v1``. Authentication uses two +headers issued at pairing time: + +* ``X-Device-Id`` — the slave's persistent UUID +* ``Authorization: Bearer `` — the per-device API key +""" +from __future__ import annotations + +import base64 +import io +import time + +import numpy as np +from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile + +from ..core.config import Config +from ..core.logger import get_logger +from .devices import SlaveDevice, SlaveDeviceStore +from .models import ConfigResponse, PairRequest, PairResponse, TurnResponse, WakeWordBlock, SlaveDeviceInfo +from .pairing import PairingCodeManager + +log = get_logger("api.routes") + +router = APIRouter(prefix="/api/v1") + +# Simple in-memory brute-force guard for /pair: per-IP attempt window. +_PAIR_MAX_ATTEMPTS = 10 +_PAIR_WINDOW_S = 300.0 +_pair_attempts: dict[str, list[float]] = {} + + +def _pair_rate_limited(client_ip: str) -> bool: + now = time.monotonic() + attempts = [t for t in _pair_attempts.get(client_ip, []) if now - t < _PAIR_WINDOW_S] + attempts.append(now) + _pair_attempts[client_ip] = attempts + return len(attempts) > _PAIR_MAX_ATTEMPTS + + +def _wake_word_block(config: Config) -> WakeWordBlock: + ww = config.wake_word + # Slaves receive the bare stem; a path on the master means nothing to them. + from pathlib import Path + name = ww.model + if name.lower().endswith(".onnx"): + name = Path(name).stem + return WakeWordBlock( + model=name, + threshold=ww.threshold, + vad_threshold=ww.vad_threshold, + silence_ms=ww.silence_ms, + ) + + +def _require_device(request: Request) -> SlaveDevice: + """FastAPI dependency: authenticate a paired slave device.""" + store: SlaveDeviceStore = request.app.state.device_store + device_id = request.headers.get("X-Device-Id", "") + auth = request.headers.get("Authorization", "") + api_key = auth.removeprefix("Bearer ").strip() if auth.startswith("Bearer ") else "" + if not device_id or not api_key or not store.verify(device_id, api_key): + raise HTTPException(status_code=401, detail="invalid device credentials") + device = store.get(device_id) + assert device is not None + client_ip = request.client.host if request.client else None + store.touch_last_seen(device_id, ip_address=client_ip) + return device + + +@router.get("/health") +async def health() -> dict: + return {"status": "ok", "service": "javis"} + + +@router.post("/pair", response_model=PairResponse) +async def pair(req: PairRequest, request: Request) -> PairResponse: + client_ip = request.client.host if request.client else "?" + if _pair_rate_limited(client_ip): + raise HTTPException(status_code=429, detail="too many pairing attempts") + pairing: PairingCodeManager = request.app.state.pairing_mgr + if not pairing.consume(req.pairing_code): + log.warning(f"pairing rejected for {client_ip} (bad/expired code)") + raise HTTPException(status_code=401, detail="invalid or expired pairing code") + + store: SlaveDeviceStore = request.app.state.device_store + api_key = store.register(req.device_id, req.device_name) + # Store the pairing IP address + store.touch_last_seen(req.device_id, ip_address=client_ip) + device = store.get(req.device_id) + config: Config = request.app.state.config + return PairResponse( + api_key=api_key, + room=device.room if device else None, + wake_word=_wake_word_block(config), + ) + + +@router.get("/config", response_model=ConfigResponse) +async def get_config( + request: Request, device: SlaveDevice = Depends(_require_device) +) -> ConfigResponse: + config: Config = request.app.state.config + return ConfigResponse(room=device.room, wake_word=_wake_word_block(config)) + + +@router.get("/wake_word/model") +async def get_wake_word_model( + request: Request, _device: SlaveDevice = Depends(_require_device) +): + """Serve the master's custom wake-word ``.onnx`` so slaves stay in sync. + + 404 when the configured model is a built-in openWakeWord model (the slave + already has those). + """ + from fastapi.responses import FileResponse + from ..audio_io.wake_word import resolve_wake_model + config: Config = request.app.state.config + try: + _arg, key, custom = resolve_wake_model( + config.wake_word.model, [config.paths.models_dir / "wakeword"]) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if custom is None: + raise HTTPException(status_code=404, detail="built-in wake word model — nothing to download") + return FileResponse(str(custom), media_type="application/octet-stream", filename=f"{key}.onnx") + + +@router.get("/slaves", response_model=list[SlaveDeviceInfo]) +async def list_slaves(request: Request, _device: SlaveDevice = Depends(_require_device)) -> list[SlaveDeviceInfo]: + """List all connected slave devices with their IP addresses. + + Requires authentication (any paired device can request this). + """ + store: SlaveDeviceStore = request.app.state.device_store + devices = store.list_devices() + return [ + SlaveDeviceInfo( + device_id=d.device_id, + name=d.name, + room=d.room, + ip_address=d.ip_address, + last_seen=d.last_seen, + last_seen_ip=d.last_seen_ip, + created_at=d.created_at, + ) + for d in devices + ] + + +@router.post("/turn", response_model=TurnResponse) +async def turn( + audio: UploadFile, + request: Request, + device: SlaveDevice = Depends(_require_device), +) -> TurnResponse: + config: Config = request.app.state.config + raw = await audio.read() + if len(raw) > config.api.max_upload_bytes: + raise HTTPException(status_code=413, detail="audio upload too large") + if not raw: + raise HTTPException(status_code=400, detail="empty audio upload") + + try: + import soundfile as sf + samples, sample_rate = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False) + if samples.ndim > 1: + samples = samples.mean(axis=1) + except Exception as e: # noqa: BLE001 + raise HTTPException(status_code=400, detail=f"could not decode audio: {e}") from e + + orch = request.app.state.orchestrator + try: + result = await orch.submit_remote_turn( + samples, int(sample_rate), room=device.room, device_name=device.name, + ) + except Exception as exc: # noqa: BLE001 + # Return a spoken error instead of a bare 500 so the slave can voice it. + log.exception(f"remote turn from {device.name!r} failed: {exc}") + phrase = config.agent.error_phrase + err_audio, err_sr = await orch.synthesize_speech(phrase) + return TurnResponse( + user_text="", + reply_text=phrase, + tts_audio_b64=_wav_b64(err_audio, err_sr), + sample_rate=err_sr if err_audio is not None else None, + error=str(exc), + ) + + return TurnResponse( + user_text=result.user_text, + reply_text=result.reply_text, + tts_audio_b64=_wav_b64(result.audio, result.sample_rate), + sample_rate=result.sample_rate if result.audio is not None else None, + ) + + +def _wav_b64(audio, sample_rate) -> str | None: + if audio is None or not sample_rate: + return None + import soundfile as sf + buf = io.BytesIO() + sf.write(buf, np.asarray(audio, dtype=np.float32), sample_rate, format="WAV", subtype="PCM_16") + return base64.b64encode(buf.getvalue()).decode("ascii") \ No newline at end of file diff --git a/src/javis/api/server.py b/src/javis/api/server.py new file mode 100644 index 0000000..971cc7b --- /dev/null +++ b/src/javis/api/server.py @@ -0,0 +1,58 @@ +"""FastAPI app factory + uvicorn runner for the slave-device API. + +Runs inside the existing asyncio loop (qasync in GUI mode) — no extra +threads or processes. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..core.config import Config +from ..core.logger import get_logger +from .devices import SlaveDeviceStore +from .pairing import PairingCodeManager + +if TYPE_CHECKING: + from fastapi import FastAPI + from ..agent.orchestrator import Orchestrator + +log = get_logger("api.server") + + +def build_app( + config: Config, + orchestrator: "Orchestrator", + device_store: SlaveDeviceStore, + pairing_mgr: PairingCodeManager, +) -> "FastAPI": + from fastapi import FastAPI + from .routes import router + + app = FastAPI(title="Javis Slave API", version="1", docs_url=None, redoc_url=None) + app.state.config = config + app.state.orchestrator = orchestrator + app.state.device_store = device_store + app.state.pairing_mgr = pairing_mgr + app.include_router(router) + return app + + +async def run_server(config: Config, app: "FastAPI") -> None: + """Serve the API until cancelled. Intended for ``asyncio.ensure_future``.""" + import uvicorn + + uv_config = uvicorn.Config( + app, + host=config.api.host, + port=config.api.port, + log_level="info", + # The qasync loop is already running; uvicorn must not install its own. + loop="none", + lifespan="off", + ) + server = uvicorn.Server(uv_config) + log.info(f"slave API listening on http://{config.api.host}:{config.api.port}") + try: + await server.serve() + except Exception as e: # noqa: BLE001 + log.error(f"slave API server stopped: {e}") diff --git a/src/javis/audio_io/__init__.py b/src/javis/audio_io/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/javis/audio_io/capture.py b/src/javis/audio_io/capture.py new file mode 100644 index 0000000..266727e --- /dev/null +++ b/src/javis/audio_io/capture.py @@ -0,0 +1,128 @@ +"""Utterance capture: VAD-segments speech from a mic stream.""" +from __future__ import annotations + +import asyncio +from typing import Optional + +import numpy as np + +from ..core.config import Config +from ..core.logger import get_logger +from .stream import MicStream +from .vad import VAD + +log = get_logger("audio.capture") + +# ── DeepFilterNet (optional noise reduction) ────────────────────────────────── +# Install with: pip install deepfilternet +# When available, each captured utterance is denoised before STT, which +# improves transcription accuracy in noisy environments. Runs on CPU so it +# doesn't compete with CUDA-based STT/TTS for VRAM. +try: + from df.enhance import enhance, init_df as _df_init_df + import torch as _df_torch + _DF_MODEL, _DF_STATE, _ = _df_init_df() + _DF_SR: int = _DF_STATE.sr() + log.info(f"DeepFilterNet loaded — denoising at {_DF_SR} Hz") + _DF_AVAILABLE = True +except Exception: + _DF_AVAILABLE = False + + +def _denoise(audio: np.ndarray, sr: int) -> np.ndarray: + """Apply DeepFilterNet speech enhancement to *audio* (float32, mono). + + Resamples to the model's native rate if needed, runs enhancement on CPU, + then resamples back. Returns a float32 numpy array at *sr*. + """ + if not _DF_AVAILABLE: + return audio + + import scipy.signal + from fractions import Fraction + + # Resample to DF native rate (48 kHz). + if sr != _DF_SR: + frac = Fraction(_DF_SR, sr).limit_denominator(1000) + audio_up = scipy.signal.resample_poly( + audio, frac.numerator, frac.denominator + ).astype(np.float32) + else: + audio_up = audio + + # DeepFilterNet expects (C, T) torch tensor on CPU. + tensor = _df_torch.from_numpy(audio_up).unsqueeze(0) + enhanced = enhance(_DF_MODEL, _DF_STATE, tensor) + audio_enh = enhanced.squeeze(0).numpy() + + # Resample back to original rate. + if sr != _DF_SR: + frac_back = Fraction(sr, _DF_SR).limit_denominator(1000) + audio_enh = scipy.signal.resample_poly( + audio_enh, frac_back.numerator, frac_back.denominator + ).astype(np.float32) + + return audio_enh + + +async def record_utterance( + config: Config, + mic: MicStream, + vad: VAD, + max_seconds: float = 20.0, + initial_grace_ms: int = 1500, + silence_ms_override: Optional[int] = None, +) -> Optional[np.ndarray]: + """Capture audio until trailing silence (`silence_ms`) is observed. + + Args: + config: Configuration object. + mic: Microphone stream. + vad: Voice Activity Detector. + max_seconds: Maximum recording duration in seconds. + initial_grace_ms: Grace period before speech is expected (ms). + silence_ms_override: Override the configured silence timeout (ms). + When set, this value is used instead of config.wake_word.silence_ms. + + Returns the recorded float32 mono numpy array at the mic's sample rate, or + None if no speech was detected within the grace period. + """ + sr = config.audio.sample_rate + silence_ms = silence_ms_override if silence_ms_override is not None else config.wake_word.silence_ms + chunk_ms = mic.chunk_ms + + silence_chunks_needed = max(1, silence_ms // chunk_ms) + grace_chunks = max(1, initial_grace_ms // chunk_ms) + max_chunks = int(max_seconds * 1000 / chunk_ms) + + vad.reset() + collected: list[np.ndarray] = [] + silent_run = 0 + seen_speech = False + chunks_seen = 0 + + async for chunk in mic.chunks(): + chunks_seen += 1 + collected.append(chunk) + result = vad.process(chunk) + + if result.is_speech: + seen_speech = True + silent_run = 0 + else: + silent_run += 1 + + if seen_speech and silent_run >= silence_chunks_needed: + break + if not seen_speech and chunks_seen >= grace_chunks: + log.debug("no speech during grace window") + return None + if chunks_seen >= max_chunks: + log.info("max utterance length reached") + break + + audio = np.concatenate(collected) if collected else np.zeros(0, dtype=np.float32) + log.info(f"utterance: {len(audio)/sr:.2f}s") + if len(audio) > 0 and _DF_AVAILABLE: + audio = _denoise(audio, sr) + return audio diff --git a/src/javis/audio_io/devices.py b/src/javis/audio_io/devices.py new file mode 100644 index 0000000..da7ac4e --- /dev/null +++ b/src/javis/audio_io/devices.py @@ -0,0 +1,45 @@ +"""Audio device discovery.""" +from __future__ import annotations + +from typing import Optional + + +def list_devices() -> list[dict]: + import sounddevice as sd + devs = sd.query_devices() + out = [] + for i, d in enumerate(devs): + out.append({ + "index": i, + "name": d["name"], + "max_input": d["max_input_channels"], + "max_output": d["max_output_channels"], + "default_samplerate": d["default_samplerate"], + "hostapi": sd.query_hostapis(d["hostapi"])["name"], + }) + return out + + +def print_devices() -> None: + for d in list_devices(): + kind = [] + if d["max_input"] > 0: kind.append(f"in:{d['max_input']}") + if d["max_output"] > 0: kind.append(f"out:{d['max_output']}") + print(f" [{d['index']:>2}] {d['name']:<40} {','.join(kind):<12} " + f"{int(d['default_samplerate'])}Hz ({d['hostapi']})") + + +def default_input_device() -> Optional[int]: + import sounddevice as sd + try: + return sd.default.device[0] + except Exception: + return None + + +def default_output_device() -> Optional[int]: + import sounddevice as sd + try: + return sd.default.device[1] + except Exception: + return None diff --git a/src/javis/audio_io/stream.py b/src/javis/audio_io/stream.py new file mode 100644 index 0000000..1af15cf --- /dev/null +++ b/src/javis/audio_io/stream.py @@ -0,0 +1,224 @@ +"""Async microphone capture and speaker playback.""" +from __future__ import annotations + +import asyncio +import time +from typing import AsyncIterator, Optional + +import numpy as np + +from ..core.config import Config +from ..core.logger import get_logger + +log = get_logger("audio.stream") + +# Warn at most once per 10 seconds when chunks are being dropped. +_DROP_WARN_INTERVAL_S = 10.0 +# Amount of audio to buffer before dropping the oldest chunk. Expressed in +# seconds (not a raw chunk count) so the latency bound stays constant if +# ``chunk_ms`` changes. +_QUEUE_BUFFER_SECONDS = 4.0 + + +class MicStream: + """Async iterator over fixed-size mic chunks (mono float32).""" + + _SENTINEL = object() + + def __init__(self, config: Config, chunk_ms: int = 30): + self.sr = config.audio.sample_rate + self.device = config.audio.input_device + self.gain = config.audio.input_gain + self.chunk_ms = chunk_ms + self.chunk_samples = int(self.sr * chunk_ms / 1000) + # Bound buffered audio to ~_QUEUE_BUFFER_SECONDS regardless of chunk_ms. + maxsize = max(8, int(_QUEUE_BUFFER_SECONDS * 1000 / chunk_ms)) + self._queue: asyncio.Queue = asyncio.Queue(maxsize=maxsize) + self._stream = None + self._loop: Optional[asyncio.AbstractEventLoop] = None + # Drop-warning rate limiter. + self._drop_count: int = 0 + self._last_drop_warn: float = 0.0 + + def _cb(self, indata, frames, time_info, status): # noqa: D401 + if status: + log.debug(f"mic status: {status}") + chunk = indata[:, 0].copy().astype(np.float32) + if self.gain != 1.0: + chunk *= self.gain + if self._loop and not self._loop.is_closed(): + self._loop.call_soon_threadsafe(self._enqueue, chunk) + + def _enqueue(self, chunk: np.ndarray) -> None: + # Drop oldest to keep mic latency bounded when the loop is busy. + if self._queue.full(): + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + pass + self._drop_count += 1 + now = time.monotonic() + if now - self._last_drop_warn >= _DROP_WARN_INTERVAL_S: + log.warning( + f"MicStream: dropped {self._drop_count} chunk(s) " + f"(~{_QUEUE_BUFFER_SECONDS:.0f}s buffer full — event loop stalled?)" + ) + self._drop_count = 0 + self._last_drop_warn = now + try: + self._queue.put_nowait(chunk) + except asyncio.QueueFull: + pass + + async def start(self) -> None: + import sounddevice as sd + self._loop = asyncio.get_running_loop() + self._stream = sd.InputStream( + samplerate=self.sr, + channels=1, + dtype="float32", + blocksize=self.chunk_samples, + device=self.device, + callback=self._cb, + ) + self._stream.start() + log.info(f"mic started @ {self.sr}Hz, chunk={self.chunk_ms}ms, device={self.device}") + + def drain(self) -> None: + """Discard all buffered chunks. Call before starting a fresh listen session + to avoid processing stale audio accumulated during STT/LLM processing.""" + drained = 0 + while True: + try: + self._queue.get_nowait() + drained += 1 + except asyncio.QueueEmpty: + break + if drained: + log.debug(f"MicStream: drained {drained} stale chunk(s)") + + def _drain_and_sentinel(self) -> None: + """Drain queue then enqueue the sentinel so chunks() always unblocks.""" + while True: + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + break + try: + self._queue.put_nowait(self._SENTINEL) + except asyncio.QueueFull: + pass + + async def stop(self) -> None: + if self._stream: + self._stream.stop() + self._stream.close() + self._stream = None + log.info("mic stopped") + # Drain first so the sentinel is never dropped when the queue is full. + if self._loop and not self._loop.is_closed(): + self._loop.call_soon_threadsafe(self._drain_and_sentinel) + + async def chunks( + self, stale_timeout: Optional[float] = None + ) -> AsyncIterator[np.ndarray]: + """Yield mic chunks. If *stale_timeout* is given and the queue stays + empty for that many seconds (possible mic disconnect), the generator + exits after logging a warning instead of blocking forever.""" + while True: + try: + if stale_timeout is not None: + chunk = await asyncio.wait_for( + self._queue.get(), timeout=stale_timeout + ) + else: + chunk = await self._queue.get() + except asyncio.TimeoutError: + log.warning( + f"MicStream: no audio for {stale_timeout:.0f}s — " + "possible microphone disconnect; re-entering wake loop" + ) + return + if chunk is self._SENTINEL: + return + yield chunk + + +def _output_channels(output_device) -> int: + """Return the channel count to use when opening an OutputStream. + + Professional audio interfaces (e.g. MOTU M Series) refuse ``channels=1`` + on Windows MME with PortAudio error -9998. We query the device's actual + max_output_channels and cap at 2 (stereo); mono audio is tiled to fill. + """ + import sounddevice as sd + try: + dev_idx = output_device if output_device is not None else sd.default.device[1] + info = sd.query_devices(dev_idx) + return min(max(1, int(info["max_output_channels"])), 2) + except Exception: + return 1 + + +async def play_pcm(samples: np.ndarray, sample_rate: int, + output_device: Optional[int] = None, + cancel_event: Optional[asyncio.Event] = None) -> None: + """Play a float32 mono numpy array. Awaitable + cancellable via event. + + Writes happen on a worker thread so the asyncio loop stays responsive + (UI updates, barge-in, MCP I/O). Cancellation aborts the device stream + immediately rather than draining the buffer. + """ + import sounddevice as sd + + samples = np.ascontiguousarray(np.asarray(samples, dtype=np.float32)) + if samples.ndim == 1: + samples = samples[:, np.newaxis] + + # Some professional audio interfaces (e.g. MOTU M Series) refuse to open + # with fewer channels than their hardware minimum. Query the actual count + # and tile mono audio to fill all channels (capped at 2 for stereo). + n_ch = _output_channels(output_device) + if samples.shape[1] < n_ch: + samples = np.repeat(samples, n_ch, axis=1) + + # ~20 ms blocks: short enough to react to barge-in, long enough to avoid + # underruns on Windows WASAPI. + block = max(256, sample_rate // 50) + stream = sd.OutputStream( + samplerate=sample_rate, + channels=n_ch, + dtype="float32", + device=output_device, + blocksize=block, + ) + + def _writer() -> None: + i = 0 + total = len(samples) + while i < total: + if cancel_event is not None and cancel_event.is_set(): + return + end = min(i + block, total) + stream.write(samples[i:end]) + i = end + + stream.start() + cancelled = False + try: + await asyncio.to_thread(_writer) + cancelled = bool(cancel_event is not None and cancel_event.is_set()) + finally: + try: + if cancelled: + stream.abort() + else: + stream.stop() + except Exception: + log.exception("stream abort failed during playback cancellation") + try: + stream.close() + except Exception: + log.exception("stream close failed during playback finalization") + if cancelled: + log.info("playback cancelled") diff --git a/src/javis/audio_io/vad.py b/src/javis/audio_io/vad.py new file mode 100644 index 0000000..fdffd5f --- /dev/null +++ b/src/javis/audio_io/vad.py @@ -0,0 +1,101 @@ +"""Voice activity detection using silero-vad (with graceful fallback).""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +import numpy as np + +from ..core.logger import get_logger + +log = get_logger("audio.vad") + + +@dataclass +class VADResult: + is_speech: bool + probability: float + + +class VAD(Protocol): + sample_rate: int + threshold: float + + def reset(self) -> None: ... + def process(self, chunk: np.ndarray) -> VADResult: ... + + +class NullVAD: + """Fallback VAD used when torch/silero is unavailable. + + Uses a simple RMS energy gate so the agent still works (with reduced + accuracy) instead of refusing to start. + """ + + def __init__(self, sample_rate: int = 16000, threshold: float = 0.5): + self.sample_rate = sample_rate + # Re-interpret the silero threshold (0..1) as an RMS gate. + # Empirically, RMS > ~0.01 in normalized float32 audio = speech. + self.threshold = max(0.005, min(0.05, threshold * 0.02)) + log.warning( + "Using NullVAD (RMS energy gate). Install torch for accurate " + "Silero VAD: python -m javis --repair" + ) + + def reset(self) -> None: + return + + def process(self, chunk: np.ndarray) -> VADResult: + if chunk.size == 0: + return VADResult(False, 0.0) + rms = float(np.sqrt(np.mean(np.square(chunk.astype(np.float32))))) + return VADResult(is_speech=rms >= self.threshold, probability=rms) + + +def make_vad(sample_rate: int = 16000, threshold: float = 0.5) -> VAD: + """Construct a VAD, falling back to NullVAD if torch/silero fails to load.""" + try: + return SileroVAD(sample_rate=sample_rate, threshold=threshold) + except Exception as e: # noqa: BLE001 + from ..core.hardware import detect_hardware, format_remediation + log.error(f"SileroVAD unavailable: {e}") + try: + log.error(format_remediation(detect_hardware(), torch_error=str(e))) + except Exception: + pass + return NullVAD(sample_rate=sample_rate, threshold=threshold) + + +class SileroVAD: + """Wraps silero-vad. Expects 16 kHz mono float32 chunks of 30 or 32 ms.""" + + def __init__(self, sample_rate: int = 16000, threshold: float = 0.5): + import torch + from silero_vad import load_silero_vad + self.sample_rate = sample_rate + self.threshold = threshold + self.model = load_silero_vad() + self._window_samples = 512 if sample_rate == 16000 else 256 + # Pre-allocated tensor to avoid per-call allocations on the hot path. + self._buf = torch.zeros(self._window_samples, dtype=torch.float32) + log.info(f"silero-vad loaded (sr={sample_rate})") + + def reset(self) -> None: + try: + self.model.reset_states() + except Exception as exc: # noqa: BLE001 + log.debug(f"SileroVAD.reset_states() failed: {exc}") + + def process(self, chunk: np.ndarray) -> VADResult: + """Process a single window; returns speech probability.""" + import torch + w = self._window_samples + n = len(chunk) + if n >= w: + self._buf.copy_(torch.from_numpy(np.ascontiguousarray(chunk[:w], dtype=np.float32))) + else: + self._buf[:n].copy_(torch.from_numpy(np.ascontiguousarray(chunk, dtype=np.float32))) + self._buf[n:].zero_() + with torch.inference_mode(): + prob = float(self.model(self._buf, self.sample_rate).item()) + return VADResult(is_speech=prob >= self.threshold, probability=prob) diff --git a/src/javis/audio_io/wake_word.py b/src/javis/audio_io/wake_word.py new file mode 100644 index 0000000..64dca1b --- /dev/null +++ b/src/javis/audio_io/wake_word.py @@ -0,0 +1,113 @@ +"""Wake word detection using openWakeWord.""" +from __future__ import annotations + +from pathlib import Path +from typing import Iterable, Optional + +import numpy as np + +from ..core.logger import get_logger + +log = get_logger("audio.wake") + +# Folder name under paths.models_dir where custom .onnx wake-word models live. +CUSTOM_MODELS_SUBDIR = "wakeword" + + +def pretrained_names() -> set[str]: + try: + import openwakeword + return set(openwakeword.MODELS.keys()) + except Exception: # noqa: BLE001 + return {"alexa", "hey_mycroft", "hey_jarvis", "hey_rhasspy", "timer", "weather"} + + +def resolve_wake_model(name: str, search_dirs: Iterable[Path] = ()) -> tuple[str, str, Optional[Path]]: + """Map a config value to ``(model_arg, prediction_key, custom_path)``. + + ``name`` may be a built-in openWakeWord name (``hey_jarvis``), a path to a + custom ``.onnx`` file, or the bare stem of a custom model located in one of + *search_dirs* (``rufus`` → ``/wakeword/rufus.onnx``). openWakeWord + keys predictions of file-based models by filename stem, so the key differs + from the argument in that case. + """ + name = (name or "hey_jarvis").strip() + p = Path(name).expanduser() + if p.suffix.lower() == ".onnx" and p.is_file(): + return str(p), p.stem, p + stem = p.stem if p.suffix else name + for d in search_dirs: + cand = Path(d) / f"{stem}.onnx" + if cand.is_file(): + return str(cand), stem, cand + if name not in pretrained_names(): + raise FileNotFoundError( + f"wake word model '{name}' is neither a built-in openWakeWord model " + f"({', '.join(sorted(pretrained_names()))}) nor a custom .onnx file in " + f"{[str(d) for d in search_dirs]} — see docs/WAKE_WORD.md") + return name, name, None + + +class WakeWordDetector: + """openWakeWord wrapper. Feeds 16 kHz int16 frames; emits scores per chunk.""" + + OWW_FRAME_SAMPLES = 1280 # openWakeWord expects 80 ms @ 16 kHz + + def __init__(self, model_name: str = "hey_jarvis", threshold: float = 0.5, + search_dirs: Iterable[Path] = ()): + from openwakeword.model import Model + from openwakeword.utils import download_models + # Ensure ONNX models are present (tflite_runtime has no Windows wheels). + try: + download_models() + except Exception as e: # noqa: BLE001 + log.warning(f"openWakeWord model download skipped: {e}") + model_arg, key, custom = resolve_wake_model(model_name, search_dirs) + self.model = Model( + wakeword_models=[model_arg], + inference_framework="onnx", + ) + # Prediction dict key (filename stem for custom models). + self.model_name = key + self.custom_path: Optional[Path] = custom + self.threshold = threshold + # Single pre-allocated 80 ms int16 frame + fill index — avoids per-chunk + # list churn and np.concatenate calls in the wake-word hot path. + self._frame = np.empty(self.OWW_FRAME_SAMPLES, dtype=np.int16) + self._fill = 0 + log.info(f"openWakeWord loaded ({model_arg}, key={key}, threshold={threshold})") + + def reset(self) -> None: + self._fill = 0 + try: + self.model.reset() + except Exception: + pass + + def feed(self, chunk_f32: np.ndarray) -> float: + """Feed an arbitrary-length mono float32 chunk; return max score across complete frames.""" + # Convert to int16 in one allocation (oww expectation). + scaled = chunk_f32 * 32768.0 + np.clip(scaled, -32768.0, 32767.0, out=scaled) + i16 = scaled.astype(np.int16, copy=False) + + max_score = 0.0 + pos = 0 + n = len(i16) + frame_size = self.OWW_FRAME_SAMPLES + while pos < n: + free = frame_size - self._fill + take = min(free, n - pos) + self._frame[self._fill:self._fill + take] = i16[pos:pos + take] + self._fill += take + pos += take + if self._fill == frame_size: + preds = self.model.predict(self._frame) + score = float(preds.get(self.model_name, 0.0)) + if score > max_score: + max_score = score + self._fill = 0 + return max_score + + def triggered(self, score: float) -> bool: + return score >= self.threshold diff --git a/src/javis/core/__init__.py b/src/javis/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/javis/core/bootstrap.py b/src/javis/core/bootstrap.py new file mode 100644 index 0000000..def0b90 --- /dev/null +++ b/src/javis/core/bootstrap.py @@ -0,0 +1,174 @@ +"""Hardware-aware dependency bootstrap / repair. + +Invoked by `python -m javis --install-deps` (fresh setup) and +`python -m javis --repair` (fix a broken environment, e.g. torch +DLL load errors on Windows). + +Design goals: +- Idempotent: safe to run repeatedly. +- Non-interactive by default; prints a clear plan first. +- Never deletes user data or venvs — only touches pip packages. +""" +from __future__ import annotations + +import subprocess +import sys +from typing import Iterable + +from .hardware import ( + HardwareInfo, + check_torch_loadable, + detect_hardware, + torch_install_command, +) +from .logger import get_logger + +log = get_logger("bootstrap") + + +# Packages whose import names differ from their pip name. +# (pip_name, import_name) +CORE_REQUIREMENTS: tuple[tuple[str, str], ...] = ( + ("pyyaml", "yaml"), + ("loguru", "loguru"), + ("httpx", "httpx"), + ("numpy", "numpy"), + ("pydantic", "pydantic"), + ("sounddevice", "sounddevice"), + ("soundfile", "soundfile"), + ("scipy", "scipy"), + ("silero-vad", "silero_vad"), + ("openwakeword", "openwakeword"), + ("onnxruntime", "onnxruntime"), + ("faster-whisper", "faster_whisper"), + ("piper-tts", "piper"), + ("mcp", "mcp"), + ("PySide6", "PySide6"), + ("qasync", "qasync"), +) + + +def _run(cmd: list[str]) -> int: + log.info("$ " + " ".join(cmd)) + proc = subprocess.run(cmd, check=False) + return proc.returncode + + +def _pip_install(pkgs: Iterable[str], *, extra_args: list[str] | None = None) -> int: + cmd = [sys.executable, "-m", "pip", "install", "--upgrade", *pkgs] + if extra_args: + cmd += extra_args + return _run(cmd) + + +def _missing_imports(reqs: Iterable[tuple[str, str]]) -> list[str]: + """Return pip names of packages that fail to import.""" + import importlib + missing: list[str] = [] + for pip_name, import_name in reqs: + try: + importlib.import_module(import_name) + except Exception: # noqa: BLE001 + missing.append(pip_name) + return missing + + +def _torch_backend_matches(info: HardwareInfo) -> bool: + """Return False when a CUDA build is expected but the installed torch lacks it.""" + try: + import torch + if info.has_nvidia_gpu and not torch.cuda.is_available(): + return False + except Exception: # noqa: BLE001 + return False + return True + + +def install_torch(info: HardwareInfo, *, force: bool = False) -> bool: + """Install / reinstall a torch wheel suitable for this hardware. + + Returns True if torch imports cleanly afterwards. + """ + ok, msg = check_torch_loadable() + if ok and not force: + if not _torch_backend_matches(info): + log.info(f"torch loaded ({msg}) but wrong variant for this hardware; reinstalling") + force = True + else: + log.info(f"torch already loadable: {msg}") + return True + + if force: + # Clean install: remove any broken existing wheel first. + _run([sys.executable, "-m", "pip", "uninstall", "-y", "torch", "torchaudio"]) + _run([sys.executable, "-m", "pip", "cache", "purge"]) + + cmd = torch_install_command(info) + rc = _run(cmd) + if rc != 0: + log.error("pip install for torch failed") + return False + + ok, msg = check_torch_loadable() + if not ok: + log.error(f"torch still not loadable after install: {msg}") + return ok + + +def install_core(force: bool = False) -> int: + """Install non-torch core deps that are missing.""" + if force: + # Reinstall everything listed. + pkgs = [name for name, _ in CORE_REQUIREMENTS] + else: + pkgs = _missing_imports(CORE_REQUIREMENTS) + if not pkgs: + log.info("core dependencies: all present") + return 0 + log.info(f"installing missing core packages: {pkgs}") + return _pip_install(pkgs) + + +def run_install(force: bool = False) -> int: + """Full install flow: hardware detection → torch → core deps.""" + info = detect_hardware() + print("=== Javis bootstrap ===") + print(f" OS: {info.os_name} ({info.arch})") + print(f" Python: {info.python_version}") + print(f" NVIDIA GPU: {info.has_nvidia_gpu}" + + (f" ({', '.join(info.gpu_names)}, driver CUDA {info.nvidia_driver_cuda})" + if info.has_nvidia_gpu else "")) + print(f" Apple MPS: {info.is_apple_silicon}") + print(f" AMD ROCm: {info.has_amd_rocm}") + print(f" Torch wheel: {info.recommended_torch_index or '(PyPI default)'}") + + if info.os_name == "Windows" and not info.has_msvc_runtime: + print() + print("WARNING: Microsoft VC++ Redistributable (x64) appears to be missing.") + print(f" Install before continuing: {info.msvc_redist_url}") + print(" Continuing anyway — torch import will likely fail.") + + # Torch first: core deps (silero-vad) would otherwise pull in a + # default-index torch wheel that gets reinstalled right after. + ok_torch = install_torch(info, force=force) + rc_core = install_core(force=force) + + print() + print("=== Bootstrap summary ===") + print(f" core deps: {'OK' if rc_core == 0 else 'FAILED'}") + print(f" torch: {'OK' if ok_torch else 'FAILED'}") + return 0 if (rc_core == 0 and ok_torch) else 1 + + +def run_repair() -> int: + """Force-reinstall torch (most common breakage on Windows).""" + info = detect_hardware() + print("=== Javis repair ===") + if info.os_name == "Windows" and not info.has_msvc_runtime: + print("VC++ Redistributable missing. Install first:") + print(f" {info.msvc_redist_url}") + print("Then run --repair again.") + return 2 + ok = install_torch(info, force=True) + print("Repair: " + ("OK" if ok else "FAILED")) + return 0 if ok else 1 diff --git a/src/javis/core/config.py b/src/javis/core/config.py new file mode 100644 index 0000000..2b19d64 --- /dev/null +++ b/src/javis/core/config.py @@ -0,0 +1,536 @@ +"""Javis configuration.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import yaml + + +@dataclass +class PathsConfig: + base_dir: Path = field(default_factory=lambda: Path.home() / ".javis") + + @property + def logs_dir(self) -> Path: return self.base_dir / "logs" + @property + def voices_dir(self) -> Path: return self.base_dir / "voices" + @property + def models_dir(self) -> Path: return self.base_dir / "models" + @property + def sessions_dir(self) -> Path: return self.base_dir / "sessions" + @property + def cache_dir(self) -> Path: return self.base_dir / "cache" + @property + def quick_commands_file(self) -> Path: return self.base_dir / "quick_commands.json" + @property + def voice_profiles_dir(self) -> Path: return self.base_dir / "voice_profiles" + @property + def slaves_file(self) -> Path: return self.base_dir / "slaves.json" + + def ensure(self) -> None: + for d in (self.base_dir, self.logs_dir, self.voices_dir, + self.models_dir, self.sessions_dir, self.cache_dir, + self.voice_profiles_dir): + d.mkdir(parents=True, exist_ok=True) + + +@dataclass +class LLMConfig: + base_url: str = "http://localhost:11434" + model: str = "llama3.1:8b" + temperature: float = 0.7 + request_timeout_seconds: int = 120 + # Context window size in tokens. Raise this when many MCP tools are + # loaded — each tool schema adds tokens. qwen3 supports 128K; + # 32768 is a safe default for 20+ tools without excessive RAM use. + num_ctx: int = 16384 + # Qwen3 thinking mode. When False, Ollama is instructed to skip the + # chain-of-thought reasoning pass (think: false in the API payload), + # which saves several seconds of latency for voice turns. + thinking_mode: bool = False + # How long (seconds) Ollama should keep the model loaded after the last + # request. Ollama's default is 5 minutes — far too short for voice + # assistant gaps — after which the model is unloaded and the next turn + # pays the full reload cost (tens of seconds for large models). + # 3600 (1 h) keeps every conversational turn snappy. "infinity" in the + # YAML is also accepted by Ollama (never unload). + keep_alive: int = 3600 + + +@dataclass +class STTConfig: + enabled: bool = True # If False, voice input (PTT / wake) is disabled. + model_size: str = "large-v3" + device: str = "auto" + compute_type: str = "float16" + language: Optional[str] = None + # Decoding: greedy (1) is ~2-3x faster than beam=5 with negligible quality + # loss on short utterances. Bump for harder audio. + beam_size: int = 1 + condition_on_previous_text: bool = False + # BatchedInferencePipeline (faster-whisper >=1.1) is ~4× faster on GPU. + # Disable if you observe decoding artefacts on short utterances. + use_batched: bool = True + batch_size: int = 8 + + +@dataclass +class TTSConfig: + backend: str = "piper" + device: str = "auto" + default_voice: str = "default" + enabled: bool = True # If False, agent runs text-only (no model load). + # ── Piper (VITS) backend ───────────────────────────────────────────────── + # Path to the trained ONNX model file. When empty the backend looks for + # ``{voices_dir}/{default_voice}/piper_model.onnx`` automatically. + piper_model: str = "" + # Speech rate multiplier: >1.0 = slower, <1.0 = faster. + piper_length_scale: float = 1.0 + # Phoneme duration noise (higher = more natural variation in timing). + piper_noise_scale: float = 0.667 + # Prosody / pitch noise (higher = more expressive but less stable). + piper_noise_w: float = 0.8 + + +@dataclass +class AudioConfig: + sample_rate: int = 16000 + input_device: Optional[int] = None + output_device: Optional[int] = None + input_gain: float = 1.0 + # Seconds of silence after the user stops speaking before a soft + # "still working" tick is played (repeats while thinking). 0 disables. + thinking_cue_delay_s: float = 2.5 + + +@dataclass +class WakeWordConfig: + enabled: bool = True + model: str = "hey_jarvis" + threshold: float = 0.5 + vad_threshold: float = 0.5 + silence_ms: int = 800 + # After a reply that ends with a question, skip wake-word detection for + # the next turn so the user can answer without the wake phrase. + followup_enabled: bool = True + # How long (seconds) to wait for speech in follow-up mode before + # re-arming the wake word. + followup_grace_s: float = 8.0 + # While Javis is speaking, keep listening for the wake word so the user + # can interrupt (barge in) playback by saying the wake phrase again. + barge_in_enabled: bool = True + + +@dataclass +class MCPConfig: + servers_config: str = "mcp_servers.json" + approval_default: str = "ask" # ask | allow | deny + call_timeout_seconds: int = 30 + # How long the GUI approval dialog waits for a click before the tool call + # is treated as denied, so a voice turn can't hang forever. 0 = wait. + approval_timeout_seconds: int = 45 + # Let the user answer an approval prompt by voice ("ja"/"nein", "yes"/"no") + # while the dialog is open. Needs STT + microphone. + voice_approval: bool = True + + +@dataclass +class APIConfig: + """HTTP API for javis-slave companion devices (LAN only, plain HTTP).""" + enabled: bool = False + host: str = "0.0.0.0" + port: int = 8765 + # Lifetime of a single-use pairing code generated in the GUI (seconds). + pairing_code_ttl_seconds: int = 300 + # Maximum accepted audio upload size (bytes). ~20 s of 16 kHz mono WAV is + # well under 1 MB; 20 MB leaves headroom for higher sample rates. + max_upload_bytes: int = 20 * 1024 * 1024 + + +@dataclass +class AgentConfig: + # Persona / behavior preset. See javis.agent.prompts.PRESETS. + preset: str = "default" # default | smart_home | concise | verbose + # Optional overrides. Precedence: file > inline > preset > built-in default. + system_prompt_inline: Optional[str] = None + system_prompt_file: Optional[str] = None + history_max_turns: int = 20 + # Maximum number of LLM ↔ tool-call iterations per turn before giving up. + max_llm_iterations: int = 5 + # TTL in seconds for the cross-turn tool result cache. Set to 0 to disable. + tool_cache_ttl_seconds: float = 60.0 + # Spoken when a voice turn fails, so users away from the screen get + # feedback instead of silence. Empty string disables. + error_phrase: str = "Entschuldigung, da ist etwas schiefgelaufen. Bitte versuch es noch einmal." + # Spoken when a tool call is waiting for approval and the window is not + # in the foreground. Empty string disables. + approval_phrase: str = "Soll ich das ausführen? Sag ja oder nein." + + +@dataclass +class MemoryConfig: + """Configuration for the background memory-extraction subsystem. + + Allows using a dedicated (typically smarter / reasoning) model for fact + extraction without changing the main conversational model. + """ + # Extraction transport: "ollama" uses the main client; "hermes" uses the + # remote OpenAI-compatible Hermes API server. + provider: str = "ollama" + # Required for Hermes, for example "http://truenas-host:30432". + base_url: Optional[str] = None + # Name of the environment variable containing Hermes API_SERVER_KEY. + api_key_env: str = "JAVIS_HERMES_API_KEY" + # Set to a model name (e.g. "qwen2.5:14b", "deepseek-r1:8b") to use a + # separate model for extraction. null = fall back to llm.model. + model: Optional[str] = None + # Temperature for extraction calls. 0.1 recommended — extraction is a + # structured JSON task where low temperature improves consistency. + temperature: float = 0.1 + # Context window for extraction. Smaller than the main llm.num_ctx is + # fine — extraction prompts are short. + num_ctx: int = 4096 + # Request timeout for extraction calls (seconds). + # Defaults to 120 so a slow reasoning model can't stall the app indefinitely. + # Increase for larger models on slow hardware; decrease for snappier operation. + # Set to null only when you want to inherit llm.request_timeout_seconds. + timeout_seconds: Optional[int] = 120 + # Maximum new facts the extraction LLM may return per turn (1–5). + # Lower values reduce duplicate smart-home patterns; raise if facts are missed. + # Editable in the Intelligence tab without restarting. + max_facts: int = 2 + # Number of already-stored entries shown to the extraction LLM so it can + # avoid re-extracting facts already saved accurately (5–50). + max_existing_for_prompt: int = 25 + + +@dataclass +class SpeakerIDConfig: + """Voice-based speaker identification configuration.""" + enabled: bool = True + # Embedding backend: "speechbrain" (ECAPA-TDNN, ~250 MB) is the default. + # "null" disables the feature at runtime without changing this flag. + model: str = "speechbrain" + # Cosine-similarity threshold for a positive identification (0–1). + # Lower values accept more false positives; higher values miss more matches. + # ECAPA-TDNN same-speaker cosine similarity in real microphone conditions + # typically ranges 0.55–0.90, so 0.65 gives a reasonable recall/precision + # trade-off. Raise towards 0.80 in quiet, controlled recording environments. + threshold: float = 0.65 + # Minimum number of recorded audio clips required to create a new profile. + min_enroll_clips: int = 3 + # When True, Javis asks for a name and auto-enrolls unknown speakers using + # the first utterance they speak after the wake word. + auto_enroll: bool = True + # When True, every confident identification (score >= threshold) automatically + # refines the stored profile with the new clip, improving accuracy over time. + auto_refine: bool = True + # Minimum margin above threshold required before a clip is used for + # auto-refinement. Only high-confidence identifications (score >= + # threshold + refine_margin) update the profile, keeping noise clips out. + refine_margin: float = 0.10 + # ── Unknown / guest speaker privileges ─────────────────────────────────── + # Applied when the speaker cannot be matched to any known voice profile. + # Controlled via the GUI "Settings → Speaker Identification" section. + # + # unknown_can_respond: If False, Javis stays silent for unrecognised voices. + # unknown_allowed_tools: "*" = all tools allowed; or a comma-separated list of + # fnmatch patterns (e.g. "spotify__*,homey__lights*"). + # unknown_denied_tools: Comma-separated fnmatch patterns that are always blocked + # (e.g. "homey__*" to deny all Homey tools for guests). + unknown_can_respond: bool = True + unknown_allowed_tools: str = "*" + unknown_denied_tools: str = "" + + +@dataclass +class TrainingConfig: + """Scheduled overnight voice-training of the currently selected voice. + + Runs ``run_pipeline`` automatically inside a daily time window so the + Piper model keeps improving while you sleep. It always resumes from the + latest checkpoint found in ``training/piper_lightning/`` (or the base + checkpoint saved in ``training/base_checkpoint.txt`` if the user set one). + + Editable in the Settings → "Voice Training" section; changes apply on the + next window because the scheduler re-reads these fields each iteration. + """ + # Master switch. When False the scheduler task is not started at all. + enabled: bool = True + # Hour (0-23) the training window opens. 3 = 03:00. + start_hour: int = 3 + # Hour (0-23) the window should close. 6 = 06:00. A run that has already + # started is not killed mid-epoch, but training will not START if the + # window is about to end (see min_remaining_minutes). + end_hour: int = 6 + # Don't START a run if fewer than this many minutes of window remain. + # Prevents a 4 a.m. launch that would run straight into the morning. + min_remaining_minutes: int = 60 + # Training hyper-parameters (defaults mirror the interactive button). + max_epochs: int = 6000 + batch_size: int = 32 + num_workers: int = 2 + accelerator: str = "gpu" # gpu | cpu + precision: str = "16-mixed" # 32 | 16-mixed | bf16-mixed + language: str = "de" # espeak-ng voice code (de, en-us, ...) + + +@dataclass +class GuiConfig: + """GUI appearance and window-behaviour settings.""" + # "dark" | "light". Both palettes cover all widgets and chat bubbles. + theme: str = "dark" + # Show the configuration dialog before the main window on every launch. + # Can be re-enabled from Tools → Startup configuration… + show_startup_dialog: bool = True + # Closing the window hides it to the system tray instead of quitting so + # wake word and timers keep running. Quit via the tray menu. + minimize_to_tray: bool = True + # System-wide push-to-talk hotkey (Windows only), e.g. "ctrl+alt+space". + # Works while the window is unfocused or hidden. Empty string disables. + global_ptt_hotkey: str = "ctrl+alt+space" + + +@dataclass +class Config: + gui: GuiConfig = field(default_factory=GuiConfig) + llm: LLMConfig = field(default_factory=LLMConfig) + stt: STTConfig = field(default_factory=STTConfig) + tts: TTSConfig = field(default_factory=TTSConfig) + audio: AudioConfig = field(default_factory=AudioConfig) + wake_word: WakeWordConfig = field(default_factory=WakeWordConfig) + mcp: MCPConfig = field(default_factory=MCPConfig) + api: APIConfig = field(default_factory=APIConfig) + agent: AgentConfig = field(default_factory=AgentConfig) + memory: MemoryConfig = field(default_factory=MemoryConfig) + speaker_id: SpeakerIDConfig = field(default_factory=SpeakerIDConfig) + training: TrainingConfig = field(default_factory=TrainingConfig) + paths: PathsConfig = field(default_factory=PathsConfig) + source_path: Optional[Path] = None + + def resolve_device(self, requested: str) -> str: + if requested != "auto": + return requested + try: + import torch + return "cuda" if torch.cuda.is_available() else "cpu" + except Exception: + return "cpu" + + +def _coerce(target: Any, key: str, val: Any, where: str, log) -> Any: + """Cast *val* to the declared type of the dataclass field, warning on failure. + + YAML users frequently quote numbers ("0.5") or write yes/no; without this + the string reaches numeric comparisons and crashes at first use. + """ + import dataclasses + if val is None: + return None + fields = {f.name: f for f in dataclasses.fields(target)} if dataclasses.is_dataclass(target) else {} + f = fields.get(key) + if f is None: + return val + current = getattr(target, key) + want = type(current) if current is not None else None + if want in (bool, int, float, str) and not isinstance(val, want): + try: + if want is bool: + if isinstance(val, str): + low = val.strip().lower() + if low in ("true", "yes", "on", "1"): + return True + if low in ("false", "no", "off", "0"): + return False + raise ValueError(val) + return bool(val) + if want is int and isinstance(val, float) and not val.is_integer(): + raise ValueError(val) + return want(val) + except (TypeError, ValueError): + log.warning(f"config: {where}={val!r} is not a {want.__name__}; keeping default {current!r}") + return current + return val + + +def _apply_section(target: Any, data: Any, section: str = "") -> None: + import logging + log = logging.getLogger("javis.config") + if not data: + return + if not isinstance(data, dict): + log.warning(f"config: section '{section}' is not a mapping (ignored)") + return + for key, val in data.items(): + if hasattr(target, key): + where = f"{section}.{key}" if section else key + setattr(target, key, _coerce(target, key, val, where, log)) + else: + where = f"{section}.{key}" if section else key + log.warning(f"config: unknown key '{where}' (ignored)") + + +_VALID_PRESETS = {"default", "smart_home", "concise", "verbose"} + + +def _clamp(val: float, lo: float, hi: float, default: float, label: str) -> float: + import logging + if lo <= val <= hi: + return val + logging.getLogger("javis.config").warning( + f"config: {label}={val} outside [{lo}, {hi}], using default {default}" + ) + return default + + +def _clamp_int(val: int, lo: int, hi: int, default: int, label: str) -> int: + import logging + if lo <= val <= hi: + return val + logging.getLogger("javis.config").warning( + f"config: {label}={val} outside [{lo}, {hi}], using default {default}" + ) + return default + + +def validate(cfg: "Config") -> None: + """Clamp/warn on out-of-range config values. Called inside load_config().""" + import logging + log = logging.getLogger("javis.config") + + cfg.wake_word.threshold = _clamp( + cfg.wake_word.threshold, 0.01, 1.0, 0.5, "wake_word.threshold") + cfg.wake_word.vad_threshold = _clamp( + cfg.wake_word.vad_threshold, 0.01, 1.0, 0.5, "wake_word.vad_threshold") + + if cfg.llm.request_timeout_seconds <= 0: + log.warning( + f"config: llm.request_timeout_seconds={cfg.llm.request_timeout_seconds} ≤ 0, " + "using default 120" + ) + cfg.llm.request_timeout_seconds = 120 + + if cfg.mcp.call_timeout_seconds <= 0: + log.warning( + f"config: mcp.call_timeout_seconds={cfg.mcp.call_timeout_seconds} ≤ 0, " + "using default 30" + ) + cfg.mcp.call_timeout_seconds = 30 + + if cfg.mcp.approval_timeout_seconds < 0: + log.warning("config: mcp.approval_timeout_seconds < 0, using 0 (no timeout)") + cfg.mcp.approval_timeout_seconds = 0 + + if cfg.audio.thinking_cue_delay_s < 0: + log.warning("config: audio.thinking_cue_delay_s < 0, using 0 (disabled)") + cfg.audio.thinking_cue_delay_s = 0.0 + + if cfg.stt.beam_size < 1: + log.warning( + f"config: stt.beam_size={cfg.stt.beam_size} < 1, using default 1") + cfg.stt.beam_size = 1 + + if cfg.agent.max_llm_iterations < 1: + log.warning( + f"config: agent.max_llm_iterations={cfg.agent.max_llm_iterations} < 1, " + "using default 5" + ) + cfg.agent.max_llm_iterations = 5 + + if cfg.agent.preset not in _VALID_PRESETS: + log.warning( + f"config: agent.preset='{cfg.agent.preset}' is unknown " + f"(valid: {sorted(_VALID_PRESETS)}), using 'default'" + ) + cfg.agent.preset = "default" + + if cfg.gui.theme not in {"dark", "light"}: + log.warning( + f"config: gui.theme='{cfg.gui.theme}' is unknown (dark|light), using 'dark'" + ) + cfg.gui.theme = "dark" + + # ── Voice training schedule ──────────────────────────────────────────── + t = cfg.training + t.start_hour = _clamp_int(t.start_hour, 0, 23, 3, "training.start_hour") + t.end_hour = _clamp_int(t.end_hour, 0, 23, 6, "training.end_hour") + if t.end_hour == t.start_hour: + log.warning( + "config: training.end_hour equals start_hour — the training window " + "would be empty; treat as a 24 h window (runs until next start)." + ) + t.min_remaining_minutes = _clamp_int( + t.min_remaining_minutes, 0, 24 * 60, 60, "training.min_remaining_minutes") + t.max_epochs = _clamp_int(t.max_epochs, 1, 1_000_000, 6000, "training.max_epochs") + t.batch_size = _clamp_int(t.batch_size, 1, 2048, 32, "training.batch_size") + t.num_workers = _clamp_int(t.num_workers, 0, 64, 2, "training.num_workers") + if t.accelerator not in {"gpu", "cpu"}: + log.warning( + f"config: training.accelerator='{t.accelerator}' is unknown " + "(gpu|cpu), using 'gpu'") + t.accelerator = "gpu" + if t.precision not in {"32", "16-mixed", "bf16-mixed"}: + log.warning( + f"config: training.precision='{t.precision}' is unknown " + "(32|16-mixed|bf16-mixed), using '16-mixed'") + t.precision = "16-mixed" + + if cfg.memory.provider not in {"ollama", "hermes"}: + log.warning( + f"config: memory.provider='{cfg.memory.provider}' is unknown, using 'ollama'" + ) + cfg.memory.provider = "ollama" + + if cfg.memory.provider == "hermes" and not cfg.memory.base_url: + raise ValueError("memory.base_url is required when memory.provider is 'hermes'") + + +def find_project_root(start: Optional[Path] = None) -> Path: + """Locate the project root by walking up until ``pyproject.toml`` is found. + + Falls back to the current working directory when no marker is found, so it + never raises regardless of how/where the package is installed. + """ + origin = (start or Path(__file__)).resolve() + for parent in (origin, *origin.parents): + if (parent / "pyproject.toml").is_file(): + return parent + return Path.cwd() + + +def load_config(path: Path) -> Config: + """Load YAML config or return defaults if missing.""" + cfg = Config() + if path.exists(): + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + + _apply_section(cfg.gui, raw.get("gui", {}), "gui") + _apply_section(cfg.llm, raw.get("llm", {}), "llm") + _apply_section(cfg.stt, raw.get("stt", {}), "stt") + _apply_section(cfg.tts, raw.get("tts", {}), "tts") + _apply_section(cfg.audio, raw.get("audio", {}), "audio") + _apply_section(cfg.wake_word, raw.get("wake_word", {}), "wake_word") + _apply_section(cfg.mcp, raw.get("mcp", {}), "mcp") + _apply_section(cfg.api, raw.get("api", {}), "api") + _apply_section(cfg.agent, raw.get("agent", {}), "agent") + _apply_section(cfg.memory, raw.get("memory", {}), "memory") + _apply_section(cfg.speaker_id, raw.get("speaker_id", {}), "speaker_id") + _apply_section(cfg.training, raw.get("training", {}), "training") + + paths = raw.get("paths", {}) or {} + base = paths.get("base_dir") + if base: + cfg.paths = PathsConfig(base_dir=Path(base).expanduser()) + + cfg.source_path = path + else: + import logging + logging.getLogger("javis.config").warning( + f"config: {path} not found — using built-in defaults") + + cfg.paths.ensure() + validate(cfg) + return cfg diff --git a/src/javis/core/config_io.py b/src/javis/core/config_io.py new file mode 100644 index 0000000..57521b8 --- /dev/null +++ b/src/javis/core/config_io.py @@ -0,0 +1,73 @@ +"""Comment-preserving writes to config.yaml. + +Every GUI path that persists a setting funnels through :func:`update_config_file` +so the user's inline documentation in config.yaml survives. ruamel.yaml keeps +comments and key order on a round-trip; PyYAML is used as a fallback when +ruamel is unavailable (comments are lost in that case, but the write still +succeeds). +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping + +try: # optional dependency — falls back to PyYAML below + from ruamel.yaml import YAML as _RuamelYAML +except Exception: # noqa: BLE001 + _RuamelYAML = None + + +def update_config_file(path: Path, updates: Mapping[str, Mapping[str, Any]]) -> None: + """Merge ``{section: {key: value}}`` into the YAML file at *path* atomically. + + Raises ``OSError`` / ``ValueError`` on failure so callers can surface the + problem to the user instead of silently dropping the change. + """ + text = path.read_text(encoding="utf-8") if path.exists() else "" + + if _RuamelYAML is not None: + yaml = _RuamelYAML() + yaml.preserve_quotes = True + yaml.width = 4096 # never re-wrap long comment/value lines + data = yaml.load(text) if text.strip() else None + if data is None: + from ruamel.yaml.comments import CommentedMap + data = CommentedMap() + for section, values in updates.items(): + sec = data.get(section) + if not isinstance(sec, Mapping): + from ruamel.yaml.comments import CommentedMap + sec = CommentedMap() + data[section] = sec + for key, value in values.items(): + sec[key] = _plain(value) + tmp = path.with_suffix(".yaml.part") + with open(tmp, "w", encoding="utf-8") as fh: + yaml.dump(data, fh) + tmp.replace(path) + return + + import yaml as _pyyaml + data = _pyyaml.safe_load(text) or {} if text.strip() else {} + if not isinstance(data, dict): + data = {} + for section, values in updates.items(): + sec = data.get(section) + if not isinstance(sec, dict): + sec = {} + data[section] = sec + for key, value in values.items(): + sec[key] = _plain(value) + tmp = path.with_suffix(".yaml.part") + tmp.write_text( + _pyyaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False), + encoding="utf-8", + ) + tmp.replace(path) + + +def _plain(value: Any) -> Any: + """Coerce Path and other non-YAML scalars to plain types.""" + if isinstance(value, Path): + return str(value) + return value diff --git a/src/javis/core/doctor.py b/src/javis/core/doctor.py new file mode 100644 index 0000000..c00b626 --- /dev/null +++ b/src/javis/core/doctor.py @@ -0,0 +1,103 @@ +"""Environment self-check (`javis --doctor`).""" +from __future__ import annotations + +import importlib +import shutil + +import httpx + +from .config import Config +from .hardware import check_ctranslate2_cuda_runtime, check_torch_loadable, detect_hardware, format_remediation +from .logger import get_logger + +log = get_logger("doctor") + + +def _check_import(module: str) -> tuple[bool, str]: + try: + importlib.import_module(module) + return True, "ok" + except Exception as e: # noqa: BLE001 + return False, f"missing: {e}" + + +def _check_torch() -> tuple[bool, str]: + ok, msg = check_torch_loadable() + return ok, msg + + +def _check_cuda() -> tuple[bool, str]: + try: + import torch + if torch.cuda.is_available(): + return True, f"CUDA ok ({torch.cuda.get_device_name(0)})" + return False, "torch present but no CUDA device" + except Exception as e: # noqa: BLE001 + return False, f"torch not loadable: {e}" + + +def _check_ollama(base_url: str, model: str) -> tuple[bool, str]: + try: + r = httpx.get(f"{base_url}/api/tags", timeout=5.0) + r.raise_for_status() + tags = [m["name"] for m in r.json().get("models", [])] + if model in tags or (":" not in model and f"{model}:latest" in tags): + return True, f"Ollama ok, model `{model}` present" + return False, f"Ollama ok but `{model}` not pulled. Run: ollama pull {model}" + except Exception as e: # noqa: BLE001 + return False, f"Ollama unreachable at {base_url}: {e}" + + +def run_doctor(config: Config) -> int: + # (name, ok, message, required) — optional rows print WARN and don't fail the check. + rows: list[tuple[str, bool, str, bool]] = [] + + hw = detect_hardware() + print("\n=== Hardware ===") + print(f" OS: {hw.os_name} ({hw.arch})") + print(f" Python: {hw.python_version}") + print(f" NVIDIA GPU: {hw.has_nvidia_gpu}" + + (f" — {', '.join(hw.gpu_names)} (driver CUDA {hw.nvidia_driver_cuda})" + if hw.has_nvidia_gpu else "")) + print(f" Apple MPS: {hw.is_apple_silicon}") + print(f" AMD ROCm: {hw.has_amd_rocm}") + if hw.os_name == "Windows": + print(f" VC++ runtime: {'OK' if hw.has_msvc_runtime else 'MISSING — install ' + hw.msvc_redist_url}") + print(f" Suggested torch wheel: {hw.recommended_torch_index or '(PyPI default)'}") + + for mod in ("sounddevice", "soundfile", "faster_whisper", + "piper", "openwakeword", "silero_vad", "mcp", "PySide6", "qasync"): + ok, msg = _check_import(mod) + rows.append((mod, ok, msg, True)) + + ok, msg = _check_torch(); rows.append(("torch", ok, msg, True)) + if hw.has_nvidia_gpu: + ok, msg = _check_cuda(); rows.append(("cuda", ok, msg, True)) + ok, msg = check_ctranslate2_cuda_runtime() + rows.append(("cublas64_12.dll", ok, msg, True)) + else: + rows.append(("cuda", True, "no NVIDIA GPU — CPU mode", True)) + ok, msg = _check_ollama(config.llm.base_url, config.llm.model) + rows.append(("ollama", ok, msg, True)) + + for tool in ("npx", "uvx"): + path = shutil.which(tool) + rows.append((tool, bool(path), + path or "not on PATH (needed for some MCP servers)", False)) + + print("\n=== Javis environment check ===") + failed = 0 + torch_failed = False + for name, ok, msg, required in rows: + flag = "OK " if ok else ("FAIL" if required else "WARN") + print(f" [{flag}] {name:<16} {msg}") + if not ok and required: + failed += 1 + if name == "torch": + torch_failed = True + print(f"\n{len(rows) - failed}/{len(rows)} checks passed.") + + if torch_failed or (hw.os_name == "Windows" and not hw.has_msvc_runtime): + print() + print(format_remediation(hw)) + return 0 if failed == 0 else 1 diff --git a/src/javis/core/events.py b/src/javis/core/events.py new file mode 100644 index 0000000..7c34866 --- /dev/null +++ b/src/javis/core/events.py @@ -0,0 +1,50 @@ +"""Lightweight asyncio pub/sub event bus.""" +from __future__ import annotations + +import asyncio +import logging +from collections import defaultdict +from typing import Any, Callable, Coroutine + +EventHandler = Callable[[Any], Coroutine[Any, Any, None]] + +_log = logging.getLogger("javis.events") + + +class EventBus: + def __init__(self) -> None: + self._subs: dict[str, list[EventHandler]] = defaultdict(list) + + def subscribe(self, event_name: str, handler: EventHandler) -> None: + self._subs[event_name].append(handler) + + def unsubscribe(self, event_name: str, handler: EventHandler) -> None: + """Remove a previously subscribed handler (no-op if absent).""" + try: + self._subs[event_name].remove(handler) + except ValueError: + pass + + async def publish(self, event_name: str, payload: Any = None) -> None: + handlers = self._subs.get(event_name) + if not handlers: + return + # Fast path: avoid gather/task overhead for the common single-subscriber case. + if len(handlers) == 1: + try: + await handlers[0](payload) + except Exception: + _log.exception(f"event handler failed for '{event_name}'") + return + results = await asyncio.gather( + *(h(payload) for h in handlers), return_exceptions=True + ) + for r in results: + # BaseException too: gather(return_exceptions=True) also captures + # CancelledError, which isinstance(r, Exception) would miss. + if isinstance(r, asyncio.CancelledError): + raise r + if isinstance(r, BaseException): + _log.error( + f"event handler failed for '{event_name}'", exc_info=r, + ) diff --git a/src/javis/core/hardware.py b/src/javis/core/hardware.py new file mode 100644 index 0000000..5b2e828 --- /dev/null +++ b/src/javis/core/hardware.py @@ -0,0 +1,309 @@ +"""Hardware and environment detection. + +Used by the bootstrap installer and the doctor to choose the right +PyTorch wheel, check for the MSVC runtime, and produce actionable +remediation hints when a dependency fails to load. + +This module must not import optional heavy dependencies (torch, +faster_whisper, …). It only uses the stdlib + shutil/subprocess. +""" +from __future__ import annotations + +import ctypes +import os +import platform +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from typing import Optional + + +# Pinned PyTorch index URLs. Update here when bumping CUDA / ROCm targets. +TORCH_INDEX_CPU = "https://download.pytorch.org/whl/cpu" +TORCH_INDEX_CUDA121 = "https://download.pytorch.org/whl/cu121" +TORCH_INDEX_CUDA124 = "https://download.pytorch.org/whl/cu124" +TORCH_INDEX_CUDA132 = "https://download.pytorch.org/whl/cu132" +TORCH_INDEX_ROCM = "https://download.pytorch.org/whl/rocm6.0" + +# CTranslate2 (used by faster-whisper for STT) is always compiled against +# CUDA 12 and requires cublas64_12.dll at runtime regardless of which CUDA +# version PyTorch was built with. Install the CUDA 12 Toolkit to satisfy +# this: https://developer.nvidia.com/cuda-12-0-1-download-archive +CTRANSLATE2_REQUIRED_CUDA_DLL = "cublas64_12.dll" + + +@dataclass +class HardwareInfo: + os_name: str = "" # 'Windows', 'Linux', 'Darwin' + arch: str = "" # 'AMD64', 'x86_64', 'arm64' + python_version: str = "" # '3.10.11' + has_nvidia_gpu: bool = False + nvidia_driver_cuda: Optional[str] = None # e.g. '12.4' + gpu_names: list[str] = field(default_factory=list) + has_amd_rocm: bool = False + is_apple_silicon: bool = False + has_msvc_runtime: bool = True # only relevant on Windows + msvc_redist_url: str = "https://aka.ms/vs/17/release/vc_redist.x64.exe" + + @property + def recommended_torch_index(self) -> str: + """Pick the best matching PyTorch wheel index for this machine.""" + if self.has_nvidia_gpu and self.nvidia_driver_cuda: + try: + major, minor = (int(p) for p in self.nvidia_driver_cuda.split(".")[:2]) + except ValueError: + major, minor = 0, 0 + # Driver must support the runtime version of the chosen wheel. + if (major, minor) >= (13, 2): + return TORCH_INDEX_CUDA132 + if (major, minor) >= (12, 4): + return TORCH_INDEX_CUDA124 + if (major, minor) >= (12, 1): + return TORCH_INDEX_CUDA121 + if self.has_amd_rocm: + return TORCH_INDEX_ROCM + # Apple Silicon → default PyPI wheel ships with MPS support. + if self.is_apple_silicon: + return "" # empty → use PyPI default index + return TORCH_INDEX_CPU + + @property + def recommended_compute_device(self) -> str: + if self.has_nvidia_gpu: + return "cuda" + if self.is_apple_silicon: + return "mps" + return "cpu" + + +def _detect_nvidia() -> tuple[bool, Optional[str], list[str]]: + """Return (has_gpu, cuda_version_from_driver, gpu_names) via nvidia-smi.""" + smi = shutil.which("nvidia-smi") + if not smi: + return False, None, [] + try: + out = subprocess.run( + [smi, "--query-gpu=name,driver_version", "--format=csv,noheader"], + capture_output=True, text=True, timeout=5, check=False, + ) + if out.returncode != 0 or not out.stdout.strip(): + return False, None, [] + names: list[str] = [] + for line in out.stdout.strip().splitlines(): + parts = [p.strip() for p in line.split(",")] + if parts: + names.append(parts[0]) + # Get the CUDA version the *driver* supports (not the toolkit). + # Only trust the explicit 'CUDA Version:' header — the first dotted + # number in the output is the driver version (e.g. 561.09), which + # would select a wildly wrong wheel. + cuda_ver: Optional[str] = None + try: + ver_out = subprocess.run( + [smi], capture_output=True, text=True, timeout=5, check=False, + ) + text = ver_out.stdout + idx = text.find("CUDA Version:") + if idx >= 0: + tail = text[idx + len("CUDA Version:"):].strip().split() + if tail: + cuda_ver = tail[0] + except Exception: + pass + return True, cuda_ver, names + except Exception: + return False, None, [] + + +def _detect_amd_gpu() -> bool: + """True when a DRM device reports the AMD PCI vendor id (0x1002).""" + import glob + for vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"): + try: + with open(vendor_file) as fh: + if fh.read().strip().lower() == "0x1002": + return True + except OSError: + continue + return shutil.which("rocm-smi") is not None + + +def _detect_msvc_runtime() -> bool: + """Best-effort check that the VC++ 2015-2022 redistributable is installed.""" + if platform.system() != "Windows": + return True + for dll in ("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"): + try: + ctypes.WinDLL(dll) # type: ignore[attr-defined] + except OSError: + return False + return True + + +def add_nvidia_dll_dirs() -> list[str]: + """Register NVIDIA pip-package DLL directories with the Windows DLL loader. + + ``nvidia-cublas-cu12``, ``nvidia-cudnn-cu12``, etc. install their CUDA + runtime DLLs (``cublas64_12.dll``, ``cudnn*.dll`` …) inside each + package's ``bin/`` sub-folder, which is *not* on ``PATH``. CTranslate2 + / faster-whisper can therefore detect CUDA but fail at first inference + with "Library cublas64_12.dll is not found or cannot be loaded". + + We call ``os.add_dll_directory()`` (Python 3.8+, Windows-only) to add + every found ``nvidia.*`` package bin dir so Windows finds them when + CTranslate2 loads its CUDA extensions. Safe to call multiple times — + already-registered dirs are deduplicated by Windows. + + Returns the list of directories that were registered. + """ + if platform.system() != "Windows": + return [] + + import importlib.util + + nvidia_subpkgs = [ + "nvidia.cublas", + "nvidia.cuda_runtime", + # nvidia.cudnn is intentionally excluded: PyTorch bundles its own cuDNN + # and putting nvidia-cudnn-cu12 on PATH causes + # CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH when the pip version + # (e.g. 9.22.x) differs from PyTorch's bundled version (e.g. 9.2.x). + # CTranslate2 does not require cuDNN for Whisper inference. + "nvidia.cufft", + "nvidia.curand", + "nvidia.cusolver", + "nvidia.cusparse", + "nvidia.nccl", + ] + + added: list[str] = [] + for pkg in nvidia_subpkgs: + try: + spec = importlib.util.find_spec(pkg) + except (ModuleNotFoundError, ValueError): + # find_spec raises ModuleNotFoundError when the parent namespace + # package (nvidia) is absent entirely, not just the subpackage. + continue + if spec is None: + continue + # nvidia.* are namespace packages (no __init__.py) so spec.origin is + # None. Use submodule_search_locations to find the package directory. + if spec.origin is not None: + pkg_dir = os.path.dirname(spec.origin) + elif spec.submodule_search_locations: + pkg_dir = list(spec.submodule_search_locations)[0] + else: + continue + for subdir in ("bin", ""): + dll_dir = os.path.join(pkg_dir, subdir) if subdir else pkg_dir + if os.path.isdir(dll_dir) and any( + f.lower().endswith(".dll") for f in os.listdir(dll_dir) + ): + # os.add_dll_directory() registers the dir for the Windows + # AddDllDirectory API (used by Python/ctypes LoadLibrary). + try: + os.add_dll_directory(dll_dir) # type: ignore[attr-defined] + except OSError: + pass + # CTranslate2 loads CUDA DLLs via its own LoadLibraryA/W + # calls which use the classic PATH search, not the newer + # AddDllDirectory list. Prepend to PATH so both mechanisms + # find the DLLs. + path_dirs = os.environ.get("PATH", "").split(os.pathsep) + if dll_dir not in path_dirs: + os.environ["PATH"] = dll_dir + os.pathsep + os.environ.get("PATH", "") + added.append(dll_dir) + break + return added + + +def check_ctranslate2_cuda_runtime() -> tuple[bool, str]: + """Check that the CUDA 12 runtime DLL required by CTranslate2 is loadable. + + CTranslate2 (used by faster-whisper for STT) is compiled against CUDA 12 + and always needs cublas64_12.dll, regardless of the PyTorch CUDA version. + Returns (ok, message). + """ + if platform.system() != "Windows": + # On Linux/macOS, libraries are found via LD_LIBRARY_PATH / rpath. + # We can't easily probe without actually loading ctranslate2. + try: + import ctranslate2 # noqa: F401 + return True, "ctranslate2 importable (assume runtime present)" + except Exception as e: # noqa: BLE001 + return False, str(e) + # Register nvidia pip-package DLL dirs first, then probe the DLL. + add_nvidia_dll_dirs() + dll = CTRANSLATE2_REQUIRED_CUDA_DLL + try: + ctypes.WinDLL(dll) # type: ignore[attr-defined] + return True, f"{dll} found" + except OSError: + return ( + False, + f"{dll} not found — install nvidia-cublas-cu12 via pip, or the " + "CUDA 12 Toolkit: https://developer.nvidia.com/cuda-12-0-1-download-archive", + ) + + +def detect_hardware() -> HardwareInfo: + info = HardwareInfo( + os_name=platform.system(), + arch=platform.machine(), + python_version=platform.python_version(), + ) + has_nv, cuda, names = _detect_nvidia() + info.has_nvidia_gpu = has_nv + info.nvidia_driver_cuda = cuda + info.gpu_names = names + + # ROCm: Linux with the ROCm stack installed *and* an AMD GPU present + # (a leftover /opt/rocm on an NVIDIA box must not select the ROCm wheel). + if platform.system() == "Linux" and os.path.isdir("/opt/rocm") and not has_nv: + info.has_amd_rocm = _detect_amd_gpu() + + if platform.system() == "Darwin" and platform.machine() == "arm64": + info.is_apple_silicon = True + + info.has_msvc_runtime = _detect_msvc_runtime() + return info + + +def check_torch_loadable() -> tuple[bool, str]: + """Try importing torch in-process; return (ok, message). + + On Windows this is the canonical 'WinError 126' early-warning probe. + """ + try: + import torch # noqa: F401 + return True, f"torch {torch.__version__} ok" + except Exception as e: # noqa: BLE001 + return False, str(e) + + +def torch_install_command(info: HardwareInfo, *, include_torchaudio: bool = True) -> list[str]: + """Return the pip command to (re)install a matching torch wheel.""" + pkgs = ["torch"] + if include_torchaudio: + pkgs.append("torchaudio") + cmd = [sys.executable, "-m", "pip", "install", "--upgrade", *pkgs] + idx = info.recommended_torch_index + if idx: + cmd += ["--index-url", idx] + return cmd + + +def format_remediation(info: HardwareInfo, torch_error: Optional[str] = None) -> str: + lines = ["Suggested remediation:"] + if info.os_name == "Windows" and not info.has_msvc_runtime: + lines.append( + f" 1. Install Microsoft VC++ Redistributable x64 " + f"({info.msvc_redist_url}) and reboot." + ) + cmd = " ".join(torch_install_command(info)) + lines.append(f" - Reinstall PyTorch matching your hardware:\n {cmd}") + if torch_error: + lines.append(f" - Original error: {torch_error}") + lines.append(" - Or run: python -m javis --repair") + return "\n".join(lines) diff --git a/src/javis/core/logger.py b/src/javis/core/logger.py new file mode 100644 index 0000000..dc86c3e --- /dev/null +++ b/src/javis/core/logger.py @@ -0,0 +1,68 @@ +"""Logging setup using loguru.""" +from __future__ import annotations + +import logging +import sys +from datetime import datetime +from typing import Optional + +from loguru import logger + +from .config import Config + +_configured = False + + +class _InterceptHandler(logging.Handler): + """Route stdlib logging records into the loguru sinks.""" + + def emit(self, record: logging.LogRecord) -> None: + try: + level: str | int = logger.level(record.levelname).name + except ValueError: + level = record.levelno + # Walk back to the original caller so loguru reports the right module. + frame, depth = logging.currentframe(), 2 + while frame is not None and frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + logger.bind(mod=record.name).opt(depth=depth, exception=record.exc_info).log( + level, record.getMessage()) + + +def setup_logger(config: Config) -> None: + global _configured + if _configured: + return + + log_dir = config.paths.logs_dir + logger.remove() + # Expose the module name as ``extra[mod]`` for the format strings below. + logger.configure(patcher=lambda record: record["extra"].setdefault("mod", record["name"])) + + logger.add( + sys.stderr, + format="{time:HH:mm:ss} | {level: <7} | " + "{extra[mod]} - {message}", + level="INFO", + colorize=True, + ) + + log_file = log_dir / f"javis_{datetime.now().strftime('%Y%m%d')}.log" + logger.add( + log_file, + format="{time:YYYY-MM-DD HH:mm:ss} | {level: <7} | {extra[mod]} - {message}", + level="DEBUG", + rotation="10 MB", + retention="14 days", + compression="zip", + ) + + # Bridge stdlib logging (timer, events, config, third-party libs) into loguru. + logging.basicConfig(handlers=[_InterceptHandler()], level=logging.INFO, force=True) + + _configured = True + + +def get_logger(module: Optional[str] = None): + return logger.bind(mod=module or "javis") diff --git a/src/javis/core/state.py b/src/javis/core/state.py new file mode 100644 index 0000000..6b990f0 --- /dev/null +++ b/src/javis/core/state.py @@ -0,0 +1,14 @@ +"""Agent state machine.""" +from __future__ import annotations + +from enum import Enum + + +class AgentState(str, Enum): + IDLE = "idle" + WAKE = "wake" # wake word detected, opening capture + LISTENING = "listening" + THINKING = "thinking" + SPEAKING = "speaking" + ENROLLING = "enrolling" # recording clips for voice profile enrollment + ERROR = "error" diff --git a/src/javis/core/timer.py b/src/javis/core/timer.py new file mode 100644 index 0000000..a320574 --- /dev/null +++ b/src/javis/core/timer.py @@ -0,0 +1,194 @@ +"""Timer service — Alexa-style countdown timers for Javis. + +`TimerService` manages asyncio-based timers that announce themselves via TTS +when they fire. Active timers are persisted to disk (absolute fire times) so +they survive a restart: still-pending timers are rescheduled on startup and +timers that already elapsed while Javis was offline are dropped. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Awaitable, Callable, Optional + +log = logging.getLogger(__name__) + +OnFireCallback = Callable[[str, str], Awaitable[None]] + + +@dataclass +class _TimerEntry: + id: str + label: str + duration_s: int + fire_at: float # absolute wall-clock epoch (time.time()) when it fires + task: "asyncio.Task[None]" + + +class TimerService: + """Manages named countdown timers backed by asyncio tasks. + + When a timer expires, *on_fire* is awaited with ``(timer_id, label)``. + The callback is responsible for synthesising and playing the alert — the + service itself does not touch audio or TTS. + + Pass *persist_path* to make active timers survive a restart. Persistence + is best-effort: failures are logged, never raised. + """ + + def __init__(self, on_fire: OnFireCallback, *, persist_path: Optional[Path] = None) -> None: + self._on_fire = on_fire + self._timers: dict[str, _TimerEntry] = {} + self._persist_path = persist_path + + # ------------------------------------------------------------------ public + + def set_timer(self, label: str, duration_s: int) -> str: + """Start a new countdown timer and return its short UUID.""" + if duration_s <= 0: + raise ValueError(f"duration_s must be positive, got {duration_s}") + if not label or not label.strip(): + label = _format_label(duration_s) + timer_id = str(uuid.uuid4())[:8] + fire_at = time.time() + duration_s + task = asyncio.ensure_future(self._run(timer_id, label, duration_s)) + self._timers[timer_id] = _TimerEntry( + id=timer_id, + label=label, + duration_s=duration_s, + fire_at=fire_at, + task=task, + ) + self._save() + log.info(f"[timer] set id={timer_id} label={label!r} duration={duration_s}s") + return timer_id + + def cancel_timer(self, timer_id: str) -> bool: + """Cancel a running timer. Returns True if it existed and was cancelled.""" + entry = self._timers.pop(timer_id, None) + if entry is None: + return False + entry.task.cancel() + self._save() + log.info(f"[timer] cancelled id={timer_id} label={entry.label!r}") + return True + + def list_timers(self) -> list[dict[str, Any]]: + """Return all active timers with seconds remaining (snapshot).""" + now = time.time() + out: list[dict[str, Any]] = [] + for entry in list(self._timers.values()): + remaining = max(0, int(entry.fire_at - now)) + out.append({ + "id": entry.id, + "label": entry.label, + "duration_s": entry.duration_s, + "seconds_remaining": remaining, + }) + return out + + def stop(self) -> None: + """Cancel every running task (call on Javis shutdown). + + The on-disk state is left intact so :meth:`restore` can reschedule the + still-pending timers on the next launch. Prefer :meth:`aclose` from + async code so the cancelled tasks are awaited (no "Task was destroyed + but it is pending" warnings at interpreter exit). + """ + for entry in list(self._timers.values()): + entry.task.cancel() + self._timers.clear() + log.info("[timer] all timer tasks cancelled (shutdown)") + + async def aclose(self) -> None: + """Cancel and await all timer tasks.""" + tasks = [e.task for e in self._timers.values()] + self.stop() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + async def restore(self) -> None: + """Reschedule persisted timers; drop any that elapsed while offline.""" + if self._persist_path is None or not self._persist_path.exists(): + return + try: + data = json.loads(self._persist_path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + log.warning(f"[timer] could not read persisted timers: {exc}") + return + now = time.time() + restored = dropped = 0 + for t in data.get("timers", []): + try: + tid = str(t["id"]) + label = str(t["label"]) + fire_at = float(t["fire_at"]) + duration_s = int(t["duration_s"]) + except (KeyError, TypeError, ValueError): + continue + remaining = fire_at - now + if remaining <= 0: + dropped += 1 + continue + task = asyncio.ensure_future(self._run(tid, label, int(round(remaining)))) + self._timers[tid] = _TimerEntry( + id=tid, label=label, duration_s=duration_s, + fire_at=fire_at, task=task, + ) + restored += 1 + self._save() + if restored or dropped: + log.info(f"[timer] restore: {restored} rescheduled, {dropped} expired-and-dropped") + + # ------------------------------------------------------------------ internal + + def _save(self) -> None: + if self._persist_path is None: + return + try: + payload = { + "version": 1, + "timers": [ + {"id": e.id, "label": e.label, + "duration_s": e.duration_s, "fire_at": e.fire_at} + for e in self._timers.values() + ], + } + self._persist_path.parent.mkdir(parents=True, exist_ok=True) + tmp = self._persist_path.with_suffix(".json.part") + tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8") + tmp.replace(self._persist_path) + except Exception: # noqa: BLE001 + log.exception("[timer] failed to persist timers") + + async def _run(self, timer_id: str, label: str, duration_s: int) -> None: + try: + await asyncio.sleep(duration_s) + except asyncio.CancelledError: + return + # Remove from the active set before firing so list_timers stays clean. + self._timers.pop(timer_id, None) + self._save() + log.info(f"[timer] fired id={timer_id} label={label!r}") + try: + await self._on_fire(timer_id, label) + except Exception: # noqa: BLE001 + log.exception(f"[timer] on_fire callback raised for id={timer_id}") + + +# ------------------------------------------------------------------ helpers + +def _format_label(seconds: int) -> str: + """Convert a raw second count into a short human-readable label.""" + if seconds < 60: + return f"{seconds}-second" + minutes, rem = divmod(seconds, 60) + if minutes < 60: + return f"{minutes}-minute" if rem == 0 else f"{minutes}m{rem}s" + hours, mins = divmod(minutes, 60) + return f"{hours}-hour" if mins == 0 else f"{hours}h{mins}m" diff --git a/src/javis/gui/__init__.py b/src/javis/gui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/javis/gui/hotkey.py b/src/javis/gui/hotkey.py new file mode 100644 index 0000000..c1a7bf8 --- /dev/null +++ b/src/javis/gui/hotkey.py @@ -0,0 +1,100 @@ +"""System-wide push-to-talk hotkey (Windows only). + +Uses ``RegisterHotKey`` so the shortcut fires even when the Javis window is +unfocused or hidden in the tray. On other platforms :func:`install_global_hotkey` +is a no-op that returns ``None``. +""" +from __future__ import annotations + +import platform +from typing import Callable, Optional + +from ..core.logger import get_logger + +log = get_logger("gui.hotkey") + +_MOD_ALT, _MOD_CTRL, _MOD_SHIFT, _MOD_WIN = 0x0001, 0x0002, 0x0004, 0x0008 +_MOD_NOREPEAT = 0x4000 +_WM_HOTKEY = 0x0312 +_HOTKEY_ID = 0x4A56 # arbitrary, "JV" + +_VK_NAMES = { + "space": 0x20, "enter": 0x0D, "return": 0x0D, "tab": 0x09, "esc": 0x1B, + "escape": 0x1B, "pause": 0x13, "insert": 0x2D, "delete": 0x2E, "home": 0x24, + "end": 0x23, "pageup": 0x21, "pagedown": 0x22, + **{f"f{i}": 0x6F + i for i in range(1, 25)}, +} + + +def parse_hotkey(spec: str) -> tuple[int, int]: + """Return ``(modifiers, virtual_key)`` for e.g. ``"ctrl+alt+space"``.""" + mods = 0 + vk: Optional[int] = None + for part in (p.strip().lower() for p in spec.split("+") if p.strip()): + if part in ("ctrl", "control"): + mods |= _MOD_CTRL + elif part == "alt": + mods |= _MOD_ALT + elif part == "shift": + mods |= _MOD_SHIFT + elif part in ("win", "super", "meta"): + mods |= _MOD_WIN + elif part in _VK_NAMES: + vk = _VK_NAMES[part] + elif len(part) == 1 and (part.isalnum()): + vk = ord(part.upper()) + else: + raise ValueError(f"unknown key '{part}' in hotkey '{spec}'") + if vk is None: + raise ValueError(f"hotkey '{spec}' has no main key") + if mods == 0: + raise ValueError(f"hotkey '{spec}' needs at least one modifier") + return mods | _MOD_NOREPEAT, vk + + +def install_global_hotkey(spec: str, callback: Callable[[], None]): + """Register *spec* system-wide; returns the filter object to keep alive, or None. + + Caller must keep a reference to the returned object for the lifetime of the + application — Qt only holds a weak reference to native event filters. + """ + if not spec or platform.system() != "Windows": + return None + import ctypes + from ctypes import wintypes + from PySide6 import QtCore + + try: + mods, vk = parse_hotkey(spec) + except ValueError as exc: + log.warning(f"global hotkey disabled: {exc}") + return None + + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + if not user32.RegisterHotKey(None, _HOTKEY_ID, mods, vk): + log.warning( + f"global hotkey '{spec}' could not be registered (already in use by " + "another application?) — window-local Ctrl+Space still works") + return None + + class _Filter(QtCore.QAbstractNativeEventFilter): + def nativeEventFilter(self, event_type, message): # noqa: N802 + if event_type in (b"windows_generic_MSG", b"windows_dispatcher_MSG"): + msg = wintypes.MSG.from_address(int(message)) + if msg.message == _WM_HOTKEY and msg.wParam == _HOTKEY_ID: + try: + callback() + except Exception: # noqa: BLE001 + log.exception("global hotkey callback failed") + return True, 0 + return False, 0 + + def uninstall(self) -> None: + user32.UnregisterHotKey(None, _HOTKEY_ID) + + flt = _Filter() + app = QtCore.QCoreApplication.instance() + if app is not None: + app.installNativeEventFilter(flt) + log.info(f"global push-to-talk hotkey registered: {spec}") + return flt diff --git a/src/javis/gui/main_window.py b/src/javis/gui/main_window.py new file mode 100644 index 0000000..b8187af --- /dev/null +++ b/src/javis/gui/main_window.py @@ -0,0 +1,5341 @@ +"""PySide6 GUI driving the orchestrator via qasync. + +Qt is imported inside `run_gui` so headless invocations don't pay the cost. +""" +from __future__ import annotations + +import asyncio +import base64 as _b64 +import json +import re +import sys +import time +from pathlib import Path + +from ..agent.orchestrator import Orchestrator +from ..core.config import Config +from ..core.config_io import update_config_file +from ..core.events import EventBus +from ..core.logger import get_logger +from ..core.state import AgentState +from ..voices.audiobook_ingest import ingest_audiobook, checkpoint_info +from ..voices.manager import VoiceManager +from .startup_dialog import show_startup_dialog +from .theme import PAL_DARK as _PAL_DARK, PAL_LIGHT as _PAL_LIGHT, THEMES as _THEME_NAMES + +log = get_logger("gui") + + +# ── VS Code Dark Modern palette — orange accent ─────────────────────────────── +_CL_BG = "#1e1e1e" # editor / main background +_CL_SURFACE = "#252526" # sidebar / panel / toolbar surface +_CL_SURFACE2 = "#2d2d2d" # inactive tabs, elevated cards +_CL_INPUT = "#3c3c3c" # text-field / input background +_CL_ACCENT = "#DA7756" # warm orange accent +_CL_ACCENT2 = "#e0886a" # accent hover +_CL_TEXT = "#cccccc" # primary text +_CL_MUTED = "#858585" # secondary / description text +_CL_BORDER = "#3c3c3c" # dividers and borders +_CL_BORDER2 = "#454545" # hover / focus border +_CL_PURPLE = "#c586c0" # VS Code purple — tool / decorator highlight +_CL_SEL = "#3d2b1f" # selection background (orange tint) +_CL_SUCCESS = "#4ec994" # success / speaking green +_CL_WARN = "#dcdcaa" # warning yellow for quick commands +_CL_ERROR = "#e06060" # error / danger red +_CL_ERROR_BG = "#3a1c1c" # error button background (dark red tint) +_CL_ERROR_BORDER = "#5a2a2a" +_CL_SUCCESS_BG = "#1e2e1e" +_CL_SUCCESS_BORDER = "#3a5a3a" +_CL_WARN_FG = "#e0a04a" # readable amber for status text +_CL_CODE_BG = "#13111e" +_CL_CODE_FG = "#c0b0e8" +_CL_TOOL_BG = "#1e1c28" +_CL_TOOL_FG = "#c0b0e8" +_CL_TOOL_MUTED = "#6a6280" +_CL_HR = "#333333" +_CL_LINK = "#555550" +_CL_QUOTE = "#aaaaaa" +_CL_USER_NAME = "#b8b8b0" + + +def _build_app_qss(pal: dict) -> str: + """Render the app-wide stylesheet for a given theme palette. + + Extracted from the inline ``app.setStyleSheet(...)`` so the light/dark + toggle can rebuild it without duplicating the rules. + """ + t = pal # short alias for readability + tick_b64 = _b64.b64encode( + b'' + b'' + ).decode() + return f""" +/* ── foundations ─────────────────────────────────────────────────────────────── */ +QMainWindow, QDialog, QWidget {{ background: {t['bg']}; color: {t['text']}; + font-family: 'Segoe UI', 'Inter', sans-serif; font-size: 10pt; }} +/* ── tab bar — active tab matches BG, inactive tabs are SURF2 (VS Code style) ── */ +QTabWidget::pane {{ border: 0; background: {t['bg']}; }} +QTabWidget::tab-bar {{ alignment: left; }} +QTabBar {{ background: {t['surface']}; border-bottom: 1px solid {t['border']}; }} +QTabBar::tab {{ background: {t['surface2']}; color: {t['muted']}; padding: 9px 22px; + border: 0; border-bottom: 2px solid transparent; + margin-right: 1px; font-size: 10pt; }} +QTabBar::tab:selected {{ background: {t['bg']}; color: {t['text']}; + border-bottom: 2px solid {t['accent']}; }} +QTabBar::tab:hover:!selected {{ background: {t['surface']}; color: {t['text']}; + border-bottom: 2px solid {t['border2']}; }} +/* ── group boxes ─────────────────────────────────────────────────────────────── */ +QGroupBox {{ border: 1px solid {t['border']}; border-radius: 6px; + margin-top: 14px; padding-top: 10px; background: {t['surface']}; }} +QGroupBox::title {{ subcontrol-origin: margin; left: 12px; padding: 0 6px; + color: {t['text']}; font-size: 9.5pt; font-weight: 600; }} +/* ── push buttons ────────────────────────────────────────────────────────────── */ +QPushButton {{ background: {t['surface']}; color: {t['text']}; + border: 1px solid {t['border']}; padding: 6px 14px; border-radius: 4px; }} +QPushButton:hover {{ background: {t['btn_hover']}; border-color: {t['border2']}; }} +QPushButton:pressed {{ background: {t['btn_pressed']}; }} +QPushButton:disabled {{ color: {t['btn_disabled']}; border-color: {t['border']}; background: {t['bg']}; }} +/* ── text inputs ─────────────────────────────────────────────────────────────── */ +QLineEdit, QPlainTextEdit, QTextEdit {{ + background: {t['input']}; color: {t['text']}; + border: 1px solid {t['border']}; border-radius: 4px; padding: 5px 8px; }} +QLineEdit:focus, QPlainTextEdit:focus {{ border-color: {t['accent']}; }} +QSpinBox, QDoubleSpinBox {{ background: {t['input']}; color: {t['text']}; + border: 1px solid {t['border']}; border-radius: 4px; padding: 3px 8px; }} +QSpinBox:focus, QDoubleSpinBox:focus {{ border-color: {t['accent']}; }} +QSpinBox::up-button, QSpinBox::down-button, +QDoubleSpinBox::up-button, QDoubleSpinBox::down-button {{ width: 16px; }} +QComboBox {{ background: {t['input']}; color: {t['text']}; + border: 1px solid {t['border']}; border-radius: 4px; padding: 4px 8px; }} +QComboBox:focus {{ border-color: {t['accent']}; }} +QComboBox::drop-down {{ border: 0; width: 18px; }} +QComboBox QAbstractItemView {{ background: {t['surface']}; color: {t['text']}; + border: 1px solid {t['border']}; + selection-background-color: {t['sel']}; }} +/* ── scrollbars ──────────────────────────────────────────────────────────────── */ +QScrollBar:vertical {{ background: {t['bg']}; width: 10px; border: 0; }} +QScrollBar::handle:vertical {{ background: {t['handle']}; border-radius: 5px; min-height: 28px; }} +QScrollBar::handle:vertical:hover {{ background: {t['handle_hover']}; }} +QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical {{ height: 0; }} +QScrollBar:horizontal {{ background: {t['bg']}; height: 10px; border: 0; }} +QScrollBar::handle:horizontal {{ background: {t['handle']}; border-radius: 5px; }} +QScrollBar::add-line:horizontal, QScrollBar::sub-line:horizontal {{ width: 0; }} +/* ── lists / trees / tables ──────────────────────────────────────────────────── */ +QTableWidget, QTreeWidget, QListWidget {{ + background: {t['surface']}; color: {t['text']}; + border: 1px solid {t['border']}; border-radius: 4px; + gridline-color: {t['surface2']}; }} +QTableWidget::item, QTreeWidget::item, QListWidget::item {{ padding: 4px 6px; }} +QTableWidget::item:hover, QTreeWidget::item:hover, QListWidget::item:hover {{ background: {t['item_hover']}; }} +QTableWidget::item:selected, QTreeWidget::item:selected, QListWidget::item:selected {{ + background: {t['sel']}; color: {t['text']}; }} +QHeaderView::section {{ background: {t['surface']}; color: {t['muted']}; padding: 5px 8px; + border: 0; border-bottom: 1px solid {t['border']}; font-size: 9pt; }} +/* ── progress bars ───────────────────────────────────────────────────────────────── */ +QProgressBar {{ background: {t['input']}; border: 1px solid {t['border']}; + border-radius: 4px; text-align: center; + color: {t['text']}; height: 8px; }} +QProgressBar::chunk {{ background: {t['accent']}; border-radius: 3px; }} +/* ── status bar ─────────────────────────────────────────────────────────────────── */ +QStatusBar {{ background: {t['surface']}; border-top: 1px solid {t['border']}; font-size: 9pt; }} +QStatusBar QLabel {{ color: {t['muted']}; }} +/* ── menu bar ───────────────────────────────────────────────────────────────────── */ +QMenuBar {{ background: {t['surface2']}; color: {t['text']}; + border-bottom: 1px solid {t['border']}; }} +QMenuBar::item {{ padding: 6px 10px; }} +QMenuBar::item:selected {{ background: {t['btn_pressed']}; }} +QMenu {{ background: {t['surface']}; color: {t['text']}; border: 1px solid {t['border']}; }} +QMenu::item {{ padding: 6px 24px; }} +QMenu::item:selected {{ background: {t['item_hover']}; }} +QMenu::separator {{ height: 1px; background: {t['border']}; margin: 4px 10px; }} +/* ── labels ─────────────────────────────────────────────────────────────────────── */ +QLabel {{ color: {t['text']}; }} +/* ── checkboxes & radios ──────────────────────────────────────────────────────────── */ +QCheckBox, QRadioButton {{ color: {t['text']}; spacing: 6px; }} +QCheckBox::indicator {{ width: 16px; height: 16px; border: 1.5px solid {t['border2']}; + border-radius: 3px; background: {t['input']}; }} +QCheckBox::indicator:unchecked:hover {{ border-color: {t['muted']}; }} +QCheckBox::indicator:checked {{ background: {t['accent']}; border-color: {t['accent']}; + image: url('data:image/svg+xml;base64,{tick_b64}'); }} +QCheckBox::indicator:checked:hover {{ background: {t['accent2']}; border-color: {t['accent2']}; + image: url('data:image/svg+xml;base64,{tick_b64}'); }} +QCheckBox::indicator:disabled {{ background: {t['surface2']}; border-color: {t['border']}; }} +QRadioButton::indicator {{ width: 14px; height: 14px; border: 1.5px solid {t['border2']}; + border-radius: 7px; background: {t['input']}; }} +QRadioButton::indicator:unchecked:hover {{ border-color: {t['muted']}; }} +QRadioButton::indicator:checked {{ background: {t['accent']}; border-color: {t['accent']}; }} +QRadioButton::indicator:checked:hover {{ background: {t['accent2']}; border-color: {t['accent2']}; }} +QRadioButton::indicator:disabled {{ background: {t['surface2']}; border-color: {t['border']}; }} +/* ── sliders ───────────────────────────────────────────────────────────────────── */ +QSlider::groove:horizontal {{ background: {t['input']}; height: 4px; border-radius: 2px; }} +QSlider::handle:horizontal {{ background: {t['accent']}; width: 14px; height: 14px; + margin: -5px 0; border-radius: 7px; }} +QSlider::sub-page:horizontal {{ background: {t['accent']}; border-radius: 2px; }} +/* ── frame separators ────────────────────────────────────────────────────────────────── */ +QFrame[frameShape="4"], QFrame[frameShape="5"] {{ color: {t['border']}; }} +/* ── tooltips ─────────────────────────────────────────────────────────────────────── */ +QToolTip {{ background: {t['surface2']}; color: {t['text']}; border: 1px solid {t['border']}; + padding: 4px 8px; border-radius: 4px; font-size: 9pt; }} +""" + + +# Number of reference clips shown per page in the refs list. +_REFS_PAGE = 20 + +# State badge colors. +_STATE_COLORS = { + "idle": "#3c3c3c", + "wake": "#c586c0", + "listening": _CL_SUCCESS, + "thinking": _CL_ACCENT, + "speaking": _CL_SUCCESS, + "enrolling": "#c586c0", + "error": _CL_ERROR, +} + + +def _level_color(v: float) -> str: + """Traffic-light color for a 0–100 level (green → yellow → red).""" + if v < 55: + return _CL_SUCCESS # green + if v < 80: + return _CL_WARN # yellow + return _CL_ERROR # red + + +# Cap the conversation view so very long sessions don't bloat memory. +_CONV_MAX_BLOCKS = 2000 + +# Keep at most this many rows in the background-tasks table. +_TASKS_MAX_ROWS = 50 + + +def _format_timer_label(seconds: int) -> str: + """Human-readable label from a raw second count.""" + if seconds < 60: + return f"{seconds}-second" + minutes, rem = divmod(seconds, 60) + if minutes < 60: + return f"{minutes}-minute" if rem == 0 else f"{minutes}m{rem}s" + hours, mins = divmod(minutes, 60) + return f"{hours}-hour" if mins == 0 else f"{hours}h{mins}m" + + +def run_gui(config: Config) -> int: + # pydantic must be imported before PySide6 to prevent a circular import + # caused by shibokensupport's hasattr hooks firing during pydantic v2 init. + import pydantic # noqa: F401 + from PySide6 import QtCore, QtGui, QtWidgets + import qasync + + # Suppress Qt CSS color parsing warnings by setting a valid default stylesheet + # This prevents spurious "Unknown color name" warnings from invalid hex colors + # in system themes or other Qt-internal stylesheets. + app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) + app.setStyle('Fusion') # Use Fusion style which is more robust + + class _MainWindow(QtWidgets.QMainWindow): + # Cross-thread-safe signals from event bus handlers + stateChanged = QtCore.Signal(str) + userText = QtCore.Signal(str) + assistantText = QtCore.Signal(str) + assistantDelta = QtCore.Signal(str) + toolCall = QtCore.Signal(str, str) + toolResult = QtCore.Signal(dict) + sttDone = QtCore.Signal(dict) + llmStatus = QtCore.Signal(dict) + ttsStatus = QtCore.Signal(dict) + mcpReady = QtCore.Signal(dict) + llmHealth = QtCore.Signal(dict) + micLevel = QtCore.Signal(dict) + wakeFired = QtCore.Signal(float) + ingestStatus = QtCore.Signal(dict) + errorEvent = QtCore.Signal(dict) + noticeEvent = QtCore.Signal(dict) + taskStart = QtCore.Signal(dict) + taskDone = QtCore.Signal(dict) + quickMatch = QtCore.Signal(dict) + quickUpdated = QtCore.Signal() + speakerIdentified = QtCore.Signal(dict) + speakerEnrolled = QtCore.Signal(dict) + speakerAskName = QtCore.Signal(dict) + enrollProgress = QtCore.Signal(dict) + timerFired = QtCore.Signal(dict) + timerUpdated = QtCore.Signal() + + def __init__(self): + super().__init__() + self.config = config # Store config as instance variable for use in methods + self.orch: Orchestrator | None = None + # Slave-device API objects (attached by run_gui after construction). + self.device_store = None + self.pairing_mgr = None + self.setWindowTitle("Javis") + self.resize(1200, 820) + + # background-task tracker: id -> {name, started, status} + self._tasks: dict[str, dict] = {} + # Quick-command card map: cmd_id -> QFrame (for flash-on-match). + self._quick_card_map: dict[str, QtWidgets.QFrame] = {} + # Voice profile panel layout refs (populated by _build_intelligence_tab). + self._voice_cards_layout: QtWidgets.QVBoxLayout | None = None + self._voice_count_lbl: QtWidgets.QLabel | None = None + self._current_speaker_lbl: QtWidgets.QLabel | None = None + # Active enrollment dialog (kept so progress signals can update it). + self._enroll_dlg: QtWidgets.QDialog | None = None + # Non-modal "new speaker" dialogs: key -> QDialog (keeps Python ref alive). + self._name_dialogs: dict[str, QtWidgets.QDialog] = {} + # Last sent user text (for Ctrl+R retry) and assistant streaming state. + self._last_sent_text: str = "" + self._assistant_buf: str = "" + # Active audiobook ingest stop handle (set to request a graceful stop). + self._ingest_stop_event: asyncio.Event | None = None + # Config widget registry — pre-initialised here so both + # _build_intelligence_tab and _build_settings_tab can register widgets + # and _save_config_to_yaml picks them all up in one pass. + self._cfg_widgets: dict[str, QtWidgets.QWidget] = {} + # Timer chip widgets keyed by timer id (for cancel buttons). + self._timer_chip_map: dict[str, QtWidgets.QFrame] = {} + # Throttle state for mic / wake bars (only repaint on meaningful change). + self._last_mic_val = -1 + self._last_wake_val = -1 + # Audiobook ingest queue tracker. + self._ingest_queue_size: int = 0 + # Streaming assistant: state for delta-to-bubble appender. + self._assistant_in_progress = False + # Error history (capped). + self._errors: list[dict] = [] + # Set by the tray "Quit" action so closeEvent really exits. + self._quit_requested = False + self._tray: QtWidgets.QSystemTrayIcon | None = None + self._tray_hint_shown = False + # Ollama model names reported at startup (None = query failed). + self._ollama_models: list[str] | None = None + + # Retheme registry: (widget, zero-arg style factory) pairs. + # _th() appends here AND applies the style now; retheme() re-applies + # all of them when the light/dark theme is toggled. This makes the + # inline (per-widget) styles — which override the app stylesheet — + # respond to theme changes, since they bake in token values. + self._themed: list[tuple] = [] + + tabs = QtWidgets.QTabWidget() + tabs.addTab(self._build_conversation_tab(), "Conversation") + tabs.addTab(self._build_status_tab(), "Status") + tabs.addTab(self._build_voices_tab(), "Voices") + intelligence_tab_idx = tabs.count() + tabs.addTab(self._build_intelligence_tab(), "Intelligence") + devices_tab_idx = tabs.count() + tabs.addTab(self._build_devices_tab(), "Devices") + tabs.addTab(self._build_settings_tab(), "Settings") + # Auto-refresh Intelligence cards whenever the tab is activated so + # changes made by the LLM (via javis__quick_* tools) are always + # visible without the user having to click the Refresh button. + tabs.currentChanged.connect( + lambda idx: self._refresh_intelligence() if idx == intelligence_tab_idx else None + ) + tabs.currentChanged.connect( + lambda idx: self._refresh_devices() if idx == devices_tab_idx else None + ) + self.setCentralWidget(tabs) + + self._build_status_bar() + + self._build_menus() + self._build_tray() + + # Wire signals + self.stateChanged.connect(self._on_state) + self.userText.connect(lambda t: self._append("You", t)) + self.assistantText.connect(self._on_assistant_text) + self.assistantDelta.connect(self._on_assistant_delta) + self.toolCall.connect(self._on_tool_call) + self.toolResult.connect(self._on_tool_result) + self.sttDone.connect(self._on_stt_done) + self.llmStatus.connect(self._on_llm_status) + self.ttsStatus.connect(self._on_tts_status) + self.mcpReady.connect(self._on_mcp_ready) + self.llmHealth.connect(self._on_llm_health) + self.micLevel.connect(self._on_mic_level) + self.wakeFired.connect(self._on_wake) + self.ingestStatus.connect(self._on_ingest) + self.errorEvent.connect(self._on_error) + self.noticeEvent.connect(self._on_notice) + self.taskStart.connect(self._on_task_start) + self.taskDone.connect(self._on_task_done) + self.quickMatch.connect(self._on_quick_match) + self.quickUpdated.connect(self._refresh_quick_cards) + self.speakerIdentified.connect(self._on_speaker_identified) + self.speakerEnrolled.connect(self._on_speaker_enrolled) + self.speakerAskName.connect(self._show_ask_name_dialog) + self.enrollProgress.connect(self._on_enroll_progress) + self.timerFired.connect(self._on_timer_fired) + self.timerUpdated.connect(self._refresh_timers) + + QtGui.QShortcut(QtGui.QKeySequence("Ctrl+Space"), self, + activated=self._push_to_talk) + QtGui.QShortcut(QtGui.QKeySequence("Esc"), self, activated=self._interrupt) + + # Re-apply every registered inline style against the active theme + # (guarantees first paint matches the selected light/dark theme). + self.retheme() + + # ─── static helper: ghost-button style used in toolbars ────────────── + # Token-based so it tracks the active theme; route through _th() when + # applying it to a widget so retheme() can refresh it on toggle. + def _ghost_qss(self): + return ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f" padding: 3px 10px; border-radius: 4px; font-size: 9.5pt; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; background: {_CL_SURFACE2}; }}" + f"QPushButton:pressed {{ background: {_CL_BORDER}; }}" + ) + + def _card_qss(self): + return ( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 10px; }}" + f"QLabel {{ border: 0; background: transparent; }}" + ) + + # ─── theme plumbing ──────────────────────────────────────────────── + def _th(self, w, factory): + """Register a theme-dependent inline style on *w* and apply it now. + + *factory* is a zero-arg callable returning the style string; it is + invoked against the CURRENT token values each time, so retheme() + can re-apply it after a light/dark toggle. Replaces the plain + ``w.setStyleSheet("...")`` calls for any style that uses theme + colors — those would otherwise be frozen at build time. + """ + self._themed.append((w, factory)) + self._apply_style(w, factory()) + + @staticmethod + def _apply_style(w, qss: str) -> None: + # QTextDocument has no setStyleSheet; its CSS goes to the default sheet. + if isinstance(w, QtGui.QTextDocument): + w.setDefaultStyleSheet(qss) + else: + w.setStyleSheet(qss) + + def retheme(self): + """Re-apply every registered inline style to the active theme. + + Called after the app palette has been switched (_apply_theme). + Base-control look (buttons/inputs/tabs) comes from the app + stylesheet which is set directly; this refreshes the custom + per-widget styles that override it. + """ + for w, factory in self._themed: + try: + self._apply_style(w, factory()) + except Exception: + pass + + def _populate_model_combo(self, ollama_models): + """Populate the model combo box with available Ollama models. + + *ollama_models* is the list returned by Ollama, or an empty list when + the query failed. Only installed models are offered so a typo can't + be persisted to config.yaml; the configured model is always listed + and flagged when Ollama doesn't know it. + """ + self._ollama_models = list(ollama_models) if ollama_models else None + current = self.config.llm.model + if self._ollama_models is None: + options = [current] + else: + options = list(dict.fromkeys( + ([current] if current not in self._ollama_models else []) + self._ollama_models)) + + # Block signals to avoid triggering _on_model_changed during update + self._model_combo.blockSignals(True) + self._model_combo.clear() + self._model_combo.addItems(options) + self._model_combo.setCurrentText(current) + missing = self._ollama_models is not None and current not in self._ollama_models + if missing: + self._model_combo.setItemData( + 0, QtGui.QBrush(QtGui.QColor(_CL_WARN)), QtCore.Qt.ItemDataRole.ForegroundRole) + self._model_combo.setItemData( + 0, f"'{current}' is not installed in Ollama — run: ollama pull {current}", + QtCore.Qt.ItemDataRole.ToolTipRole) + self._model_combo.blockSignals(False) + if missing: + self._on_notice({"where": "llm", + "message": f"Model '{current}' not found in Ollama — run: ollama pull {current}"}) + elif self._ollama_models is None: + self._model_combo.setToolTip( + "Ollama could not be reached — model list unavailable.\n" + "Edit llm.model in Settings if you need to change it.") + + # ------------------------- builders ------------------------- + + def _build_conversation_tab(self): + w = QtWidgets.QWidget() + self._th(w, lambda: f"QWidget {{ background: {_CL_BG}; }}") + v = QtWidgets.QVBoxLayout(w) + v.setContentsMargins(0, 0, 0, 0) + v.setSpacing(0) + + # ── compact top toolbar ─────────────────────────────────────────── + toolbar = QtWidgets.QWidget() + toolbar.setFixedHeight(40) + self._th(toolbar, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-bottom: 1px solid {_CL_BORDER}; }}" + )) + tb = QtWidgets.QHBoxLayout(toolbar) + tb.setContentsMargins(10, 0, 10, 0) + tb.setSpacing(2) + + btn_clear = QtWidgets.QPushButton("Clear") + btn_clear.setToolTip("Clear conversation view (does not reset LLM memory)") + btn_clear.clicked.connect(self._clear_conversation) + btn_copy = QtWidgets.QPushButton("Copy") + btn_copy.setToolTip("Copy entire conversation to clipboard") + btn_copy.clicked.connect(self._copy_conversation) + btn_save = QtWidgets.QPushButton("Save…") + btn_save.setToolTip("Save conversation transcript as Markdown / text") + btn_save.clicked.connect(self._save_conversation) + btn_reset_mem = QtWidgets.QPushButton("Reset memory") + btn_reset_mem.setToolTip("Forget conversation history sent to the LLM") + btn_reset_mem.clicked.connect(self._reset_memory) + for _b in (btn_clear, btn_copy, btn_save, btn_reset_mem): + self._th(_b, self._ghost_qss) + _b.setFixedHeight(28) + tb.addWidget(_b) + + tb.addStretch(1) + + # ── model selector ──────────────────────────────────────────────── + _model_lbl = QtWidgets.QLabel("Model:") + self._th(_model_lbl, lambda: f"color: {_CL_MUTED}; font-size: 9.5pt; background: transparent;") + tb.addWidget(_model_lbl) + + # Create the combo box - models will be populated after orchestrator starts + self._model_combo = QtWidgets.QComboBox() + + # Populate with fallback models initially; will be updated in _bootstrap() when LLM starts + self._populate_model_combo([]) + + self._model_combo.setToolTip( + "Switch the active LLM model.\n" + "Takes effect on the next request — no restart needed.\n" + "Change persisted to config.yaml automatically." + ) + self._model_combo.setFixedHeight(26) + self._model_combo.setMinimumWidth(160) + self._th(self._model_combo, lambda: ( + f"QComboBox {{ background: {_CL_SURFACE2}; color: {_CL_TEXT}; " + f" border: 1px solid {_CL_BORDER}; border-radius: 4px; " + f" padding: 2px 8px; font-size: 9.5pt; }}" + f"QComboBox:hover {{ border-color: {_CL_BORDER2}; }}" + f"QComboBox::drop-down {{ border: 0; width: 18px; }}" + f"QComboBox QAbstractItemView {{ background: {_CL_SURFACE}; " + f" color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; " + f" selection-background-color: {_CL_ACCENT}; outline: 0; }}" + )) + + def _on_model_changed(name: str) -> None: + if not name or name == config.llm.model: + return + config.llm.model = name + # Sync Status-tab label + self._lbl_llm_model.setText(name) + # Sync Settings-tab QLineEdit so a subsequent Save captures it + _w = self._cfg_widgets.get("llm.model") + if isinstance(_w, QtWidgets.QLineEdit): + _w.setText(name) + # Persist to config.yaml immediately + _persist_llm_model(name) + + def _persist_llm_model(name: str) -> None: + if config.source_path is None: + return + try: + update_config_file(config.source_path, {"llm": {"model": name}}) + except Exception as exc: + log.warning(f"[gui] failed to persist llm.model: {exc}") + self._on_notice({"where": "config", + "message": f"Model switched for this session only — config.yaml not writable: {exc}"}) + + self._model_combo.currentTextChanged.connect(_on_model_changed) + tb.addWidget(self._model_combo) + + self._show_tools = QtWidgets.QCheckBox("Tools") + self._show_tools.setChecked(False) + self._show_tools.setToolTip("Show tool calls and results in conversation") + self._th(self._show_tools, lambda: f"color: {_CL_MUTED}; font-size: 9.5pt;") + tb.addWidget(self._show_tools) + + self._auto_approve_btn = QtWidgets.QPushButton("🔓 Auto-approve") + self._auto_approve_btn.setCheckable(True) + self._auto_approve_btn.setChecked(False) + self._auto_approve_btn.setFixedHeight(28) + self._auto_approve_btn.setToolTip( + "Automatically approve every MCP tool call without a dialog prompt.\n" + "WARNING: tools can modify files and control devices." + ) + self._th(self._auto_approve_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_BORDER};" + f" color: {_CL_MUTED}; padding: 3px 10px; border-radius: 4px; font-size: 9.5pt; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; background: {_CL_SURFACE2}; }}" + f"QPushButton:checked {{ background: {_CL_SUCCESS_BG}; color: {_CL_SUCCESS};" + f" border-color: {_CL_SUCCESS_BORDER}; font-weight: 600; }}" + f"QPushButton:checked:hover {{ background: {_CL_SUCCESS_BORDER}; }}" + )) + tb.addWidget(self._auto_approve_btn) + + btn_sysinfo = QtWidgets.QPushButton("ℹ") + btn_sysinfo.setToolTip("Open live system information panel") + btn_sysinfo.setFixedHeight(28) + btn_sysinfo.setFixedWidth(28) + self._th(btn_sysinfo, self._ghost_qss) + btn_sysinfo.clicked.connect(self._show_system_info) + tb.addWidget(btn_sysinfo) + v.addWidget(toolbar) + + # ── chat view ───────────────────────────────────────────────────── + self.conv_view = QtWidgets.QTextEdit() + self.conv_view.setReadOnly(True) + self._th(self.conv_view, lambda: ( + f"QTextEdit {{ background: {_CL_BG}; color: {_CL_TEXT}; " + f"font-family: 'Segoe UI', 'Inter', sans-serif; font-size: 11pt; " + f"padding: 8px 16px; border: 0; }}" + )) + # Re-applied on retheme via _th so link colour follows the accent. + self._th(self.conv_view.document(), lambda: ( + f"body {{ background: {_CL_BG}; }}" + "table { margin: 0; border-spacing: 0; }" + f"a {{ color: {_CL_ACCENT}; }}" + )) + v.addWidget(self.conv_view, 1) + + # ── active timers bar (hidden when no timers running) ─────────────── + self._timer_bar = QtWidgets.QFrame() + self._th(self._timer_bar, lambda: ( + f"QFrame {{ background: {_CL_SUCCESS_BG}; border-top: 1px solid {_CL_SUCCESS_BORDER}; " + f"border-bottom: 1px solid {_CL_BORDER}; }}" + )) + self._timer_bar.setFixedHeight(36) + self._timer_bar.hide() + _tbar_lay = QtWidgets.QHBoxLayout(self._timer_bar) + _tbar_lay.setContentsMargins(12, 0, 12, 0) + _tbar_lay.setSpacing(8) + _timer_icon = QtWidgets.QLabel("⏱") + self._th(_timer_icon, lambda: ( + f"color: {_CL_SUCCESS}; font-size: 13pt; background: transparent; border: 0;" + )) + _tbar_lay.addWidget(_timer_icon) + self._timer_chips_layout = QtWidgets.QHBoxLayout() + self._timer_chips_layout.setSpacing(6) + _tbar_lay.addLayout(self._timer_chips_layout) + _tbar_lay.addStretch(1) + v.addWidget(self._timer_bar) + + # ── bottom input panel ──────────────────────────────────────────── + input_panel = QtWidgets.QWidget() + self._th(input_panel, lambda: f"QWidget {{ background: {_CL_BG}; border-top: 1px solid {_CL_BORDER}; }}") + ipv = QtWidgets.QVBoxLayout(input_panel) + ipv.setContentsMargins(16, 10, 16, 12) + ipv.setSpacing(8) + + # PTT + Interrupt row + btn_row = QtWidgets.QHBoxLayout() + btn_row.setSpacing(8) + self._ptt_btn = QtWidgets.QPushButton("🎤 Push to Talk (Ctrl+Space)") + self._ptt_btn.setMinimumHeight(36) + self._th(self._ptt_btn, lambda: ( + f"QPushButton {{ background: {_CL_SURFACE2}; color: {_CL_TEXT}; " + f" border: 1px solid {_CL_BORDER}; padding: 0 20px; border-radius: 18px; font-size: 10pt; }}" + f"QPushButton:hover {{ border-color: {_CL_ACCENT}; background: {_CL_SURFACE}; }}" + f"QPushButton:pressed {{ background: {_CL_BORDER}; }}" + )) + self._ptt_btn.clicked.connect(self._push_to_talk) + + self._interrupt_btn = QtWidgets.QPushButton("⏹ Stop") + self._interrupt_btn.setMinimumHeight(36) + self._interrupt_btn.setEnabled(False) + self._th(self._interrupt_btn, lambda: ( + f"QPushButton {{ background: {_CL_ERROR_BG}; color: {_CL_ERROR}; " + f" border: 1px solid {_CL_ERROR_BORDER}; padding: 0 16px; border-radius: 18px; font-size: 10pt; }}" + f"QPushButton:hover {{ background: {_CL_ERROR_BORDER}; }}" + f"QPushButton:disabled {{ background: {_CL_SURFACE}; color: {_CL_MUTED}; " + f" border-color: {_CL_BORDER}; }}" + )) + self._interrupt_btn.clicked.connect(self._interrupt) + btn_row.addWidget(self._ptt_btn, 1) + btn_row.addWidget(self._interrupt_btn) + ipv.addLayout(btn_row) + + # Text input card — rounded, Claude-style + input_card = QtWidgets.QFrame() + self._th(input_card, lambda: ( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f" border-radius: 12px; }}" + )) + card_lay = QtWidgets.QHBoxLayout(input_card) + card_lay.setContentsMargins(14, 4, 6, 4) + card_lay.setSpacing(8) + + self._text_input = QtWidgets.QLineEdit() + self._text_input.setPlaceholderText("Message Javis… (Ctrl+L)") + self._text_input.setMinimumHeight(38) + self._th(self._text_input, lambda: ( + f"QLineEdit {{ background: transparent; border: 0; color: {_CL_TEXT}; " + f" font-size: 11pt; padding: 0; }}" + f"QLineEdit::placeholder {{ color: {_CL_MUTED}; }}" + )) + self._text_input.returnPressed.connect(self._send_text) + + self._speak_reply_cb = QtWidgets.QCheckBox("Speak") + self._speak_reply_cb.setChecked(True) + self._speak_reply_cb.setToolTip("Speak the assistant reply via TTS") + self._th(self._speak_reply_cb, lambda: f"color: {_CL_MUTED}; font-size: 9.5pt;") + + voice_lbl = QtWidgets.QLabel("Voice:") + self._th(voice_lbl, lambda: f"color: {_CL_MUTED}; font-size: 9.5pt;") + self._voice_combo = QtWidgets.QComboBox() + self._voice_combo.setFixedWidth(120) + self._voice_combo.setFixedHeight(30) + self._voice_combo.setToolTip("TTS voice profile") + tts_on = config.tts.enabled + voice_lbl.setEnabled(tts_on) + self._voice_combo.setEnabled(tts_on) + self._populate_voice_combo() + self._voice_combo.currentTextChanged.connect(self._on_voice_changed) + + self._send_btn = QtWidgets.QPushButton("↑") + self._send_btn.setFixedSize(36, 36) + self._send_btn.setToolTip("Send (Enter)") + self._th(self._send_btn, lambda: ( + f"QPushButton {{ background: {_CL_ACCENT}; color: white; border: 0; " + f" border-radius: 8px; font-size: 15pt; font-weight: bold; }}" + f"QPushButton:hover {{ background: {_CL_ACCENT2}; }}" + f"QPushButton:pressed {{ background: {_CL_ACCENT2}; }}" + )) + self._send_btn.clicked.connect(self._send_text) + + card_lay.addWidget(self._text_input, 1) + card_lay.addWidget(self._speak_reply_cb) + card_lay.addWidget(voice_lbl) + card_lay.addWidget(self._voice_combo) + card_lay.addWidget(self._send_btn) + ipv.addWidget(input_card) + + v.addWidget(input_panel) + + QtGui.QShortcut(QtGui.QKeySequence("Ctrl+L"), self, + activated=lambda: self._text_input.setFocus()) + QtGui.QShortcut(QtGui.QKeySequence("Ctrl+R"), self, + activated=self._retry_last) + return w + + # --- conversation toolbar handlers --- + + def _clear_conversation(self): + self.conv_view.clear() + self._assistant_in_progress = False + + def _copy_conversation(self): + QtWidgets.QApplication.clipboard().setText(self.conv_view.toPlainText()) + + def _save_conversation(self): + path, _ = QtWidgets.QFileDialog.getSaveFileName( + self, "Save conversation", "javis_transcript.md", + "Markdown (*.md);;Text (*.txt)", + ) + if not path: + return + try: + Path(path).write_text(self.conv_view.toPlainText(), encoding="utf-8") + except OSError as e: + QtWidgets.QMessageBox.warning(self, "Save failed", str(e)) + + def _reset_memory(self): + if self.orch is None: + return + self.orch.conv.clear() + self._append("system", "(conversation memory cleared)") + + def _build_status_tab(self): + w = QtWidgets.QWidget() + grid = QtWidgets.QGridLayout(w) + + # --- Agent box --- + agent_box = QtWidgets.QGroupBox("Agent") + af = QtWidgets.QFormLayout(agent_box) + self.lbl_state_big = QtWidgets.QLabel("idle") + self.lbl_state_big.setStyleSheet( + "font-size: 20pt; font-weight: bold; padding: 4px;") + af.addRow("State:", self.lbl_state_big) + self.lbl_last_turn = QtWidgets.QLabel("—") + af.addRow("Last turn:", self.lbl_last_turn) + self.lbl_last_error = QtWidgets.QLabel("—") + self.lbl_last_error.setWordWrap(True) + self._th(self.lbl_last_error, lambda: f"color: {_CL_ERROR};") + af.addRow("Last error:", self.lbl_last_error) + grid.addWidget(agent_box, 0, 0) + + # --- Audio box --- + audio_box = QtWidgets.QGroupBox("Audio / Wake word") + af2 = QtWidgets.QFormLayout(audio_box) + self.mic_bar = QtWidgets.QProgressBar() + self.mic_bar.setRange(0, 100); self.mic_bar.setValue(0) + self.mic_bar.setTextVisible(False) + af2.addRow("Mic level:", self.mic_bar) + self.wake_bar = QtWidgets.QProgressBar() + self.wake_bar.setRange(0, 100); self.wake_bar.setValue(0) + self.wake_bar.setFormat("%v%") + af2.addRow("Wake score:", self.wake_bar) + self.lbl_last_wake = QtWidgets.QLabel("never") + af2.addRow("Last wake:", self.lbl_last_wake) + grid.addWidget(audio_box, 0, 1) + + # --- STT box --- + stt_box = QtWidgets.QGroupBox("Speech-to-text") + sf = QtWidgets.QFormLayout(stt_box) + if config.stt.enabled: + sf.addRow("Model:", QtWidgets.QLabel( + f"{config.stt.model_size} ({config.stt.device}/{config.stt.compute_type})")) + else: + lbl_disabled = QtWidgets.QLabel("disabled — text-only input") + self._th(lbl_disabled, lambda: f"color: {_CL_WARN_FG};") + sf.addRow("Model:", lbl_disabled) + self.lbl_stt_last = QtWidgets.QLabel("—") + self.lbl_stt_last.setWordWrap(True) + sf.addRow("Last result:", self.lbl_stt_last) + self.lbl_stt_timing = QtWidgets.QLabel("—") + sf.addRow("Timing:", self.lbl_stt_timing) + grid.addWidget(stt_box, 1, 0) + + # --- LLM box --- + llm_box = QtWidgets.QGroupBox("LLM (Ollama)") + lf = QtWidgets.QFormLayout(llm_box) + lf.addRow("URL:", QtWidgets.QLabel(config.llm.base_url)) + self._lbl_llm_model = QtWidgets.QLabel(config.llm.model) + lf.addRow("Model:", self._lbl_llm_model) + self.lbl_llm_health = QtWidgets.QLabel("checking…") + lf.addRow("Health:", self.lbl_llm_health) + self.lbl_llm_phase = QtWidgets.QLabel("idle") + lf.addRow("Phase:", self.lbl_llm_phase) + self.lbl_llm_timing = QtWidgets.QLabel("—") + lf.addRow("Last latency:", self.lbl_llm_timing) + grid.addWidget(llm_box, 1, 1) + + # --- TTS box --- + tts_box = QtWidgets.QGroupBox("Text-to-speech") + tf = QtWidgets.QFormLayout(tts_box) + tf.addRow("Backend:", QtWidgets.QLabel(config.tts.backend)) + tf.addRow("Default voice:", QtWidgets.QLabel(config.tts.default_voice)) + self.lbl_tts_phase = QtWidgets.QLabel("idle") + tf.addRow("Phase:", self.lbl_tts_phase) + self.tts_progress = QtWidgets.QProgressBar() + self.tts_progress.setRange(0, 1); self.tts_progress.setValue(0) + tf.addRow("Sentence:", self.tts_progress) + self.lbl_tts_timing = QtWidgets.QLabel("—") + tf.addRow("Last synth:", self.lbl_tts_timing) + grid.addWidget(tts_box, 2, 0) + + # --- MCP box --- + mcp_box = QtWidgets.QGroupBox("MCP servers / tools") + mlay = QtWidgets.QVBoxLayout(mcp_box) + self.lbl_mcp_summary = QtWidgets.QLabel("not started") + mlay.addWidget(self.lbl_mcp_summary) + self.mcp_tree = QtWidgets.QTreeWidget() + self.mcp_tree.setHeaderLabels(["Server / Tool", "Detail"]) + self.mcp_tree.setColumnWidth(0, 240) + mlay.addWidget(self.mcp_tree, 1) + grid.addWidget(mcp_box, 2, 1) + + # --- Background tasks --- + tasks_box = QtWidgets.QGroupBox("Background tasks") + tlay = QtWidgets.QVBoxLayout(tasks_box) + self.task_table = QtWidgets.QTableWidget(0, 3) + self.task_table.setHorizontalHeaderLabels(["Task", "Status", "Elapsed"]) + self.task_table.horizontalHeader().setStretchLastSection(True) + self.task_table.verticalHeader().setVisible(False) + self.task_table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers) + tlay.addWidget(self.task_table, 1) + grid.addWidget(tasks_box, 3, 0, 1, 2) + + grid.setRowStretch(2, 1) + grid.setRowStretch(3, 1) + + # Periodic elapsed-time refresh + self._tick_timer = QtCore.QTimer(self) + self._tick_timer.setInterval(500) + self._tick_timer.timeout.connect(self._refresh_task_elapsed) + self._tick_timer.timeout.connect(self._refresh_timers) + self._tick_timer.start() + + return w + + def _build_voices_tab(self): + w = QtWidgets.QWidget() + v = QtWidgets.QVBoxLayout(w) + v.addWidget(QtWidgets.QLabel("Voice profiles:")) + self.voice_list = QtWidgets.QListWidget() + v.addWidget(self.voice_list, 1) + self.voice_list.currentItemChanged.connect( + lambda cur, _prev: ( + self._refresh_refs( + cur.data(QtCore.Qt.UserRole) if cur else None + ), + self._refresh_piper_status( + cur.data(QtCore.Qt.UserRole) if cur else None + ), + ) + ) + + row = QtWidgets.QHBoxLayout() + new_btn = QtWidgets.QPushButton("New profile") + new_btn.clicked.connect(self._new_voice) + ingest_btn = QtWidgets.QPushButton("Ingest audiobook…") + ingest_btn.clicked.connect(self._ingest_audiobook) + row.addWidget(new_btn); row.addWidget(ingest_btn) + row.addStretch(1) + export_btn = QtWidgets.QPushButton("Export…") + export_btn.setToolTip("Pack the selected profile into a .javisvoice archive") + export_btn.clicked.connect(self._export_voice) + import_btn = QtWidgets.QPushButton("Import…") + import_btn.setToolTip("Import a .javisvoice archive as a new voice profile") + import_btn.clicked.connect(self._import_voice) + delete_btn = QtWidgets.QPushButton("Delete") + delete_btn.setToolTip("Permanently delete the selected voice profile") + delete_btn.clicked.connect(self._delete_voice) + row.addWidget(export_btn); row.addWidget(import_btn); row.addWidget(delete_btn) + v.addLayout(row) + + self.voice_status = QtWidgets.QLabel("") + v.addWidget(self.voice_status) + self.ingest_progress = QtWidgets.QProgressBar() + self.ingest_progress.setRange(0, 1); self.ingest_progress.setValue(0) + self.ingest_progress.hide() + v.addWidget(self.ingest_progress) + self.ingest_stop_btn = QtWidgets.QPushButton("Stop ingest") + self.ingest_stop_btn.setToolTip("Save a checkpoint and stop after the current segment") + self.ingest_stop_btn.hide() + self.ingest_stop_btn.clicked.connect(self._stop_ingest) + v.addWidget(self.ingest_stop_btn) + # Onboarding hint — visible when profiles are missing or have no ref clips. + self.voice_hint = QtWidgets.QLabel() + self.voice_hint.setWordWrap(True) + self._th(self.voice_hint, lambda: f"color: {_CL_ACCENT}; padding: 4px;") + self.voice_hint.hide() + v.addWidget(self.voice_hint) + + # ── Reference clips ────────────────────────────────────────────── + refs_box = QtWidgets.QGroupBox("Reference clips (selected profile)") + self._th(refs_box, lambda: ( + f"QGroupBox {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 8px; margin-top: 10px; padding: 8px 10px 8px 10px; }}" + f"QGroupBox::title {{ subcontrol-origin: margin; left: 12px; padding: 0 4px; " + f"color: {_CL_TEXT}; font-weight: 600; font-size: 9.5pt; }}" + )) + refs_lay = QtWidgets.QVBoxLayout(refs_box) + refs_lay.setSpacing(4) + refs_lay.setContentsMargins(6, 8, 6, 6) + + self.refs_list = QtWidgets.QListWidget() + self.refs_list.setFixedHeight(110) + self.refs_list.setSelectionMode( + QtWidgets.QAbstractItemView.SelectionMode.SingleSelection + ) + self.refs_list.setToolTip( + "WAV clips used for voice training input (ingest source).\n" + "Use 'Ingest audiobook…' to populate automatically." + ) + self._th(self.refs_list, lambda: ( + f"QListWidget {{ background: {_CL_BG}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 5px; font-size: 9pt; }}" + f"QListWidget::item:selected {{ background: {_CL_ACCENT}; color: white; }}" + )) + refs_lay.addWidget(self.refs_list) + + refs_hint = QtWidgets.QLabel( + "Select a profile above to manage its reference clips." + ) + self._th(refs_hint, lambda: f"color: {_CL_MUTED}; font-size: 8.5pt; background: transparent;") + self.refs_hint_lbl = refs_hint + refs_lay.addWidget(refs_hint) + + def _btn_ss(): + return ( + f"QPushButton {{ background: {_CL_SURFACE2}; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_TEXT}; border-radius: 4px; padding: 3px 10px; font-size: 9pt; }}" + f"QPushButton:hover {{ background: {_CL_SURFACE}; border-color: {_CL_BORDER2}; }}" + ) + self._refs_load_more_btn = QtWidgets.QPushButton("Load more clips…") + self._th(self._refs_load_more_btn, _btn_ss) + self._refs_load_more_btn.clicked.connect(self._refs_load_more) + self._refs_load_more_btn.hide() + refs_lay.addWidget(self._refs_load_more_btn) + + refs_btn_row = QtWidgets.QHBoxLayout() + refs_btn_row.setSpacing(6) + self._refs_add_btn = QtWidgets.QPushButton("Add clip\u2026") + self._refs_add_btn.setToolTip("Copy WAV file(s) into this profile\u2019s refs/ folder") + self._th(self._refs_add_btn, _btn_ss) + self._refs_add_btn.clicked.connect(self._add_ref) + self._refs_play_btn = QtWidgets.QPushButton("\u25b6 Play") + self._refs_play_btn.setToolTip("Preview the selected reference clip") + self._th(self._refs_play_btn, _btn_ss) + self._refs_play_btn.clicked.connect(self._play_ref) + self._refs_del_btn = QtWidgets.QPushButton("Delete") + self._refs_del_btn.setToolTip("Permanently remove the selected clip from refs/") + self._th(self._refs_del_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_ERROR_BORDER}; " + f"color: {_CL_ERROR}; border-radius: 4px; padding: 3px 10px; font-size: 9pt; }}" + f"QPushButton:hover {{ background: {_CL_ERROR_BG}; border-color: {_CL_ERROR}; }}" + )) + self._refs_del_btn.clicked.connect(self._delete_ref) + refs_btn_row.addWidget(self._refs_add_btn) + refs_btn_row.addWidget(self._refs_play_btn) + refs_btn_row.addStretch(1) + refs_btn_row.addWidget(self._refs_del_btn) + refs_lay.addLayout(refs_btn_row) + + v.addWidget(refs_box) + + # ── Piper model training ───────────────────────────────────────── + piper_box = QtWidgets.QGroupBox("Piper model (selected profile)") + self._th(piper_box, lambda: ( + f"QGroupBox {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 8px; margin-top: 10px; padding: 8px 10px 8px 10px; }}" + f"QGroupBox::title {{ subcontrol-origin: margin; left: 12px; padding: 0 4px; " + f"color: {_CL_TEXT}; font-weight: 600; font-size: 9.5pt; }}" + )) + piper_lay = QtWidgets.QVBoxLayout(piper_box) + piper_lay.setSpacing(4) + piper_lay.setContentsMargins(6, 8, 6, 6) + + self._piper_status_lbl = QtWidgets.QLabel("Select a profile above.") + self._th(self._piper_status_lbl, lambda: f"color: {_CL_MUTED}; font-size: 8.5pt; background: transparent;") + piper_lay.addWidget(self._piper_status_lbl) + + piper_btn_row = QtWidgets.QHBoxLayout() + piper_btn_row.setSpacing(6) + def _btn_ss2(): + return ( + f"QPushButton {{ background: {_CL_SURFACE2}; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_TEXT}; border-radius: 4px; padding: 3px 10px; font-size: 9pt; }}" + f"QPushButton:hover {{ background: {_CL_SURFACE}; border-color: {_CL_BORDER2}; }}" + ) + self._piper_train_btn = QtWidgets.QPushButton("Train model\u2026") + self._piper_train_btn.setToolTip( + "Launch Piper VITS training for this profile (requires piper-train + espeak-ng)" + ) + self._th(self._piper_train_btn, _btn_ss2) + self._piper_train_btn.clicked.connect(self._train_piper_model) + piper_btn_row.addWidget(self._piper_train_btn) + + self._piper_snapshot_btn = QtWidgets.QPushButton("Export snapshot\u2026") + self._piper_snapshot_btn.setToolTip( + "Export the current best training checkpoint to ONNX now\n" + "(can be done while training is paused, without waiting for it to finish)" + ) + self._th(self._piper_snapshot_btn, _btn_ss2) + self._piper_snapshot_btn.clicked.connect(self._export_piper_snapshot) + piper_btn_row.addWidget(self._piper_snapshot_btn) + + self._piper_activate_btn = QtWidgets.QPushButton("Use as active voice") + self._piper_activate_btn.setToolTip( + "Set this profile as the active TTS voice in config.yaml\n" + "and reload the Piper backend (takes effect immediately)" + ) + self._th(self._piper_activate_btn, _btn_ss2) + self._piper_activate_btn.clicked.connect(self._activate_piper_voice) + piper_btn_row.addWidget(self._piper_activate_btn) + + piper_btn_row.addStretch(1) + piper_lay.addLayout(piper_btn_row) + + v.addWidget(piper_box) + + # ── TTS test ───────────────────────────────────────────────────── + sep = QtWidgets.QFrame() + sep.setFrameShape(QtWidgets.QFrame.HLine) + self._th(sep, lambda: f"color:{_CL_BORDER};") + v.addWidget(sep) + + test_box = QtWidgets.QGroupBox("Test TTS") + tb = QtWidgets.QVBoxLayout(test_box) + tb.setSpacing(6) + + # Row 1: voice selector + sel_row = QtWidgets.QHBoxLayout() + sel_row.addWidget(QtWidgets.QLabel("Voice:")) + self._tts_test_combo = QtWidgets.QComboBox() + self._tts_test_combo.setToolTip("Voice profile to use for the test") + self._tts_test_combo.setEnabled(config.tts.enabled) + sel_row.addWidget(self._tts_test_combo, 1) + tb.addLayout(sel_row) + + # Row 2: text input + button + inp_row = QtWidgets.QHBoxLayout() + self._tts_test_input = QtWidgets.QLineEdit() + self._tts_test_input.setPlaceholderText("Enter text to speak…") + self._tts_test_input.setEnabled(config.tts.enabled) + self._tts_test_input.returnPressed.connect(self._run_tts_test) + inp_row.addWidget(self._tts_test_input, 1) + self._tts_test_btn = QtWidgets.QPushButton("▶ Test") + self._tts_test_btn.setEnabled(config.tts.enabled) + self._tts_test_btn.setToolTip("Synthesize and play the text with the selected voice") + self._tts_test_btn.clicked.connect(self._run_tts_test) + inp_row.addWidget(self._tts_test_btn) + tb.addLayout(inp_row) + + # Status label + self._tts_test_status = QtWidgets.QLabel( + "TTS disabled — set tts.enabled in config.yaml" if not config.tts.enabled + else "Ready" + ) + self._th(self._tts_test_status, lambda: f"color: {_CL_MUTED}; font-size: 10pt;") + tb.addWidget(self._tts_test_status) + + v.addWidget(test_box) + + self._refresh_voices() + return w + + # ------------------------- devices tab ------------------------- + + @staticmethod + def _lan_ip() -> str: + """Best-effort LAN IP of this machine (for display only).""" + import socket + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("8.8.8.8", 80)) # no packets sent for UDP connect + return s.getsockname()[0] + finally: + s.close() + except Exception: + log.debug("[network] failed to determine local IP address, using 127.0.0.1") + return "127.0.0.1" + + def _build_devices_tab(self): + w = QtWidgets.QWidget() + self._th(w, lambda: f"QWidget {{ background: {_CL_BG}; }}") + outer = QtWidgets.QVBoxLayout(w) + outer.setContentsMargins(0, 0, 0, 0) + outer.setSpacing(0) + + # ── toolbar ────────────────────────────────────────────────────── + toolbar = QtWidgets.QWidget() + toolbar.setFixedHeight(44) + self._th(toolbar, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-bottom: 1px solid {_CL_BORDER}; }}" + )) + tb = QtWidgets.QHBoxLayout(toolbar) + tb.setContentsMargins(16, 0, 16, 0) + tb.setSpacing(10) + hdr = QtWidgets.QLabel("📱 Slave Devices") + self._th(hdr, lambda: f"color: {_CL_TEXT}; font-weight: 700; font-size: 11pt; background: transparent;") + tb.addWidget(hdr) + tb.addStretch(1) + btn_refresh = QtWidgets.QPushButton("⟳ Refresh") + self._th(btn_refresh, self._ghost_qss) + btn_refresh.setCursor(QtCore.Qt.PointingHandCursor) + btn_refresh.clicked.connect(self._refresh_devices) + tb.addWidget(btn_refresh) + btn_pair = QtWidgets.QPushButton("＋ Generate pairing code") + btn_pair.setCursor(QtCore.Qt.PointingHandCursor) + self._th(btn_pair, lambda: ( + f"QPushButton {{ background: {_CL_ACCENT}; color: #1e1e1e; font-weight: 600; " + f" border: 0; border-radius: 4px; padding: 5px 14px; }}" + f"QPushButton:hover {{ background: {_CL_ACCENT2}; }}" + )) + btn_pair.clicked.connect(self._generate_pairing_code) + tb.addWidget(btn_pair) + outer.addWidget(toolbar) + + body = QtWidgets.QVBoxLayout() + body.setContentsMargins(16, 12, 16, 12) + body.setSpacing(10) + + # ── API status / pairing-code display card ─────────────────────── + self._pair_card = QtWidgets.QFrame() + self._th(self._pair_card, lambda: ( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f" border-radius: 10px; }}" + )) + pc = QtWidgets.QVBoxLayout(self._pair_card) + pc.setContentsMargins(16, 12, 16, 12) + pc.setSpacing(4) + if config.api.enabled: + api_line = (f"API listening on http://{self._lan_ip()}:{config.api.port}" + f" (bind: {config.api.host})") + else: + api_line = ("API disabled — set api.enabled: true in config.yaml " + "and restart to allow slave devices to connect.") + self._api_status_lbl = QtWidgets.QLabel(api_line) + self._th(self._api_status_lbl, lambda: ( + f"color: {_CL_MUTED}; background: transparent; border: 0; font-size: 9.5pt;")) + pc.addWidget(self._api_status_lbl) + self._pair_code_lbl = QtWidgets.QLabel("") + self._th(self._pair_code_lbl, lambda: ( + f"color: {_CL_ACCENT}; background: transparent; border: 0; " + f"font-size: 30pt; font-weight: 800; letter-spacing: 8px;")) + self._pair_code_lbl.setAlignment(QtCore.Qt.AlignCenter) + self._pair_code_lbl.hide() + pc.addWidget(self._pair_code_lbl) + self._pair_hint_lbl = QtWidgets.QLabel("") + self._th(self._pair_hint_lbl, lambda: ( + f"color: {_CL_MUTED}; background: transparent; border: 0; font-size: 9pt;")) + self._pair_hint_lbl.setAlignment(QtCore.Qt.AlignCenter) + self._pair_hint_lbl.hide() + pc.addWidget(self._pair_hint_lbl) + body.addWidget(self._pair_card) + + # ── device list ────────────────────────────────────────────────── + scroll = QtWidgets.QScrollArea() + scroll.setWidgetResizable(True) + self._th(scroll, lambda: "QScrollArea { border: 0; background: transparent; }") + inner = QtWidgets.QWidget() + inner.setStyleSheet("background: transparent;") + self._devices_layout = QtWidgets.QVBoxLayout(inner) + self._devices_layout.setContentsMargins(0, 0, 0, 0) + self._devices_layout.setSpacing(8) + self._devices_layout.addStretch(1) + scroll.setWidget(inner) + body.addWidget(scroll, 1) + + outer.addLayout(body, 1) + return w + + def _generate_pairing_code(self): + if self.pairing_mgr is None: + return + if not config.api.enabled: + QtWidgets.QMessageBox.information( + self, "API disabled", + "Enable the slave API first: api.enabled: true in config.yaml, " + "then restart Javis.") + return + code = self.pairing_mgr.generate() + ttl_min = max(1, config.api.pairing_code_ttl_seconds // 60) + self._pair_code_lbl.setText(code) + self._pair_code_lbl.show() + self._pair_hint_lbl.setText( + f"Enter this code on the slave device together with " + f"{self._lan_ip()}:{config.api.port} — valid for {ttl_min} min, single use." + ) + self._pair_hint_lbl.show() + + def _refresh_devices(self): + if self._devices_layout is None or self.device_store is None: + return + # Clear existing cards (keep trailing stretch). + while self._devices_layout.count() > 1: + item = self._devices_layout.takeAt(0) + if item.widget(): + item.widget().deleteLater() + devices = self.device_store.list_devices() + if not devices: + empty = QtWidgets.QLabel( + "No slave devices paired yet.\nGenerate a pairing code and enter it " + "in the javis-slave app on the device.") + empty.setAlignment(QtCore.Qt.AlignCenter) + self._th(empty, lambda: ( + f"color: {_CL_MUTED}; background: transparent; font-size: 10pt; padding: 30px;")) + self._devices_layout.insertWidget(0, empty) + return + for i, dev in enumerate(devices): + self._devices_layout.insertWidget(i, self._make_device_card(dev)) + + def _make_device_card(self, dev): + card = QtWidgets.QFrame() + self._th(card, lambda: ( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f" border-radius: 10px; }}" + f"QFrame:hover {{ border-color: {_CL_BORDER2}; }}" + )) + h = QtWidgets.QHBoxLayout(card) + h.setContentsMargins(14, 10, 14, 10) + h.setSpacing(12) + + info = QtWidgets.QVBoxLayout() + info.setSpacing(2) + name_lbl = QtWidgets.QLabel(f"🖥 {dev.name}") + self._th(name_lbl, lambda: ( + f"color: {_CL_TEXT}; background: transparent; border: 0; " + f"font-weight: 700; font-size: 10.5pt;")) + info.addWidget(name_lbl) + seen = dev.last_seen or "never" + meta_lbl = QtWidgets.QLabel(f"id: {dev.device_id} · last seen: {seen}") + self._th(meta_lbl, lambda: ( + f"color: {_CL_MUTED}; background: transparent; border: 0; font-size: 8.5pt;")) + info.addWidget(meta_lbl) + h.addLayout(info, 1) + + room_edit = QtWidgets.QLineEdit(dev.room or "") + room_edit.setPlaceholderText("Room (unassigned)") + room_edit.setFixedWidth(180) + self._th(room_edit, lambda: ( + f"QLineEdit {{ background: {_CL_INPUT}; color: {_CL_TEXT}; border: 1px solid " + f"{_CL_BORDER}; border-radius: 4px; padding: 4px 8px; }}" + f"QLineEdit:focus {{ border-color: {_CL_ACCENT}; }}" + )) + h.addWidget(room_edit) + + btn_save = QtWidgets.QPushButton("Save room") + self._th(btn_save, self._ghost_qss) + btn_save.setCursor(QtCore.Qt.PointingHandCursor) + + def _save_room(): + self.device_store.assign_room(dev.device_id, room_edit.text()) + self.statusBar().showMessage( + f"Room for '{dev.name}' set to '{room_edit.text().strip() or '—'}'", 4000) + + btn_save.clicked.connect(_save_room) + h.addWidget(btn_save) + + btn_revoke = QtWidgets.QPushButton("Revoke") + btn_revoke.setCursor(QtCore.Qt.PointingHandCursor) + self._th(btn_revoke, lambda: ( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_ERROR_BORDER}; " + f" color: {_CL_ERROR}; padding: 3px 12px; border-radius: 4px; }}" + f"QPushButton:hover {{ background: {_CL_ERROR_BG}; }}" + )) + + def _revoke(): + ret = QtWidgets.QMessageBox.question( + self, "Revoke device", + f"Revoke access for '{dev.name}'?\nThe device must be re-paired to reconnect.") + if ret == QtWidgets.QMessageBox.Yes: + self.device_store.revoke(dev.device_id) + self._refresh_devices() + + btn_revoke.clicked.connect(_revoke) + h.addWidget(btn_revoke) + return card + + def _build_settings_tab(self): + from ..agent.prompts import PRESETS + + w = QtWidgets.QWidget() + self._th(w, lambda: f"QWidget {{ background: {_CL_BG}; }}") + outer = QtWidgets.QVBoxLayout(w) + outer.setContentsMargins(0, 0, 0, 0) + outer.setSpacing(0) + + # ── toolbar ────────────────────────────────────────────────────── + toolbar = QtWidgets.QWidget() + toolbar.setFixedHeight(44) + self._th(toolbar, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-bottom: 1px solid {_CL_BORDER}; }}" + )) + tb = QtWidgets.QHBoxLayout(toolbar) + tb.setContentsMargins(16, 0, 16, 0) + tb.setSpacing(10) + hdr = QtWidgets.QLabel("⚙ Configuration") + self._th(hdr, lambda: f"color: {_CL_TEXT}; font-weight: 700; font-size: 11pt; background: transparent;") + self._cfg_save_lbl = QtWidgets.QLabel("") + self._th(self._cfg_save_lbl, lambda: ( + f"color: {_CL_SUCCESS}; font-size: 9pt; background: transparent;" + )) + save_btn = QtWidgets.QPushButton("💾 Save to config.yaml") + self._th(save_btn, lambda: ( + f"QPushButton {{ background: {_CL_ACCENT}; color: #fff; border: 0; " + f"border-radius: 8px; padding: 6px 18px; font-weight: 600; }}" + f"QPushButton:hover {{ background: {_CL_ACCENT2}; }}" + )) + save_btn.clicked.connect(self._save_config_to_yaml) + tb.addWidget(hdr) + tb.addStretch() + tb.addWidget(self._cfg_save_lbl) + tb.addWidget(save_btn) + outer.addWidget(toolbar) + + # ── scrollable body ────────────────────────────────────────────── + scroll = QtWidgets.QScrollArea() + scroll.setWidgetResizable(True) + scroll.setFrameShape(QtWidgets.QFrame.Shape.NoFrame) + self._th(scroll, lambda: f"QScrollArea {{ background: {_CL_BG}; border: 0; }}") + body = QtWidgets.QWidget() + self._th(body, lambda: f"QWidget {{ background: {_CL_BG}; }}") + body_lay = QtWidgets.QVBoxLayout(body) + body_lay.setContentsMargins(20, 20, 20, 20) + body_lay.setSpacing(14) + scroll.setWidget(body) + outer.addWidget(scroll, 1) + + # ── shared widget styles (zero-arg factories, re-evaluated on retheme) ── + def _grp_qss(): + return ( + f"QGroupBox {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 10px; margin-top: 12px; padding: 10px 14px 10px 14px; }}" + f"QGroupBox::title {{ subcontrol-origin: margin; left: 14px; padding: 0 6px; " + f"color: {_CL_TEXT}; font-weight: 700; font-size: 10pt; }}" + f"QLabel {{ color: {_CL_TEXT}; background: transparent; font-size: 9.5pt; }}" + f"QLineEdit, QSpinBox, QDoubleSpinBox, QComboBox {{ " + f" background: {_CL_BG}; color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; " + f" border-radius: 5px; padding: 3px 8px; font-size: 9.5pt; min-width: 60px; }}" + f"QCheckBox {{ color: {_CL_TEXT}; background: transparent; font-size: 9.5pt; }}" + ) + + def _btn_sm(): + return ( + f"QPushButton {{ background: {_CL_SURFACE2}; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_TEXT}; border-radius: 5px; padding: 3px 10px; font-size: 9pt; }}" + f"QPushButton:hover {{ background: {_CL_BORDER}; }}" + ) + + def _sec(title: str) -> tuple: + grp = QtWidgets.QGroupBox(title) + self._th(grp, _grp_qss) + form = QtWidgets.QFormLayout(grp) + form.setVerticalSpacing(8) + form.setHorizontalSpacing(20) + form.setFieldGrowthPolicy( + QtWidgets.QFormLayout.FieldGrowthPolicy.ExpandingFieldsGrow + ) + return grp, form + + def _w_line(key, val, placeholder="", tip="") -> QtWidgets.QLineEdit: + ed = QtWidgets.QLineEdit("" if val is None else str(val)) + if placeholder: + ed.setPlaceholderText(placeholder) + if tip: + ed.setToolTip(tip) + ed.setMinimumWidth(200) + self._cfg_widgets[key] = ed + return ed + + def _w_spin(key, val, lo, hi, step=1, tip="") -> QtWidgets.QSpinBox: + sp = QtWidgets.QSpinBox() + sp.setRange(lo, hi) + sp.setSingleStep(step) + sp.setValue(int(val) if val is not None else lo) + if tip: + sp.setToolTip(tip) + sp.setMinimumWidth(110) + self._cfg_widgets[key] = sp + return sp + + def _w_dspin(key, val, lo, hi, step=0.05, dec=2, tip="") -> QtWidgets.QDoubleSpinBox: + sp = QtWidgets.QDoubleSpinBox() + sp.setRange(lo, hi) + sp.setSingleStep(step) + sp.setDecimals(dec) + sp.setValue(float(val) if val is not None else lo) + if tip: + sp.setToolTip(tip) + sp.setMinimumWidth(110) + self._cfg_widgets[key] = sp + return sp + + def _w_check(key, val, tip="") -> QtWidgets.QCheckBox: + cb = QtWidgets.QCheckBox() + cb.setChecked(bool(val)) + if tip: + cb.setToolTip(tip) + self._cfg_widgets[key] = cb + return cb + + def _w_combo(key, choices, val, tip="") -> QtWidgets.QComboBox: + cb = QtWidgets.QComboBox() + for c in choices: + cb.addItem(str(c)) + idx = next((i for i, c in enumerate(choices) if str(c) == str(val)), 0) + cb.setCurrentIndex(idx) + if tip: + cb.setToolTip(tip) + cb.setMinimumWidth(150) + self._cfg_widgets[key] = cb + return cb + + R = " ↺" # requires restart indicator + L = " ⚡" # live indicator + + # ── LLM ───────────────────────────────────────────────────────── + grp, form = _sec("🤖 LLM") + form.addRow(f"Base URL{R}", + _w_line("llm.base_url", config.llm.base_url, + tip="Ollama API base URL (restart required)")) + + llm_row = QtWidgets.QHBoxLayout() + llm_row.setSpacing(6) + llm_edit = _w_line("llm.model", config.llm.model, + tip="Ollama model name, e.g. qwen2.5:7b") + self._llm_check_lbl = QtWidgets.QLabel("") + self._th(self._llm_check_lbl, lambda: ( + f"color: {_CL_SUCCESS}; font-size: 8.5pt; background: transparent;" + )) + llm_test = QtWidgets.QPushButton("Test") + llm_test.setFixedWidth(55) + self._th(llm_test, _btn_sm) + def _check_llm(): + if not self.orch: + return + async def _do(): + ok = await self.orch.llm.health() + c = _CL_SUCCESS if ok else _CL_ERROR + t = "✓ reachable" if ok else "✗ unreachable" + self._llm_check_lbl.setText( + f"{t}") + asyncio.ensure_future(_do()) + llm_test.clicked.connect(_check_llm) + llm_row.addWidget(llm_edit, 1) + llm_row.addWidget(llm_test) + llm_row.addWidget(self._llm_check_lbl) + form.addRow(f"Model{R}", llm_row) + + form.addRow("Temperature", + _w_dspin("llm.temperature", config.llm.temperature, 0.0, 2.0, 0.05, + tip="Sampling temperature — 0 = deterministic, 2 = very random")) + form.addRow(f"Timeout (s){L}", + _w_spin("llm.request_timeout_seconds", + config.llm.request_timeout_seconds, 10, 3600, 10)) + form.addRow(f"Context (tokens){R}", + _w_spin("llm.num_ctx", config.llm.num_ctx, 512, 131072, 512, + tip="LLM context window in tokens — raise for many MCP tools")) + body_lay.addWidget(grp) + + # ── STT ───────────────────────────────────────────────────────── + grp, form = _sec("🎤 Speech-to-Text (STT)") + form.addRow(f"Enabled{R}", + _w_check("stt.enabled", config.stt.enabled, + tip="Disable for text-only mode (no mic needed)")) + form.addRow(f"Model{R}", + _w_combo("stt.model_size", + ["tiny", "tiny.en", "base", "base.en", "small", "small.en", + "medium", "medium.en", "large-v2", "large-v3", + "distil-large-v2", "distil-large-v3"], + config.stt.model_size, + tip="Whisper model — larger is more accurate but slower to load")) + form.addRow(f"Device{R}", + _w_combo("stt.device", + ["auto", "cpu", "cuda", "cuda:0", "cuda:1", "cuda:2", "cuda:3"], + config.stt.device)) + form.addRow(f"Compute type{R}", + _w_combo("stt.compute_type", + ["float16", "int8_float32", "float32", "int8", "int16"], + config.stt.compute_type, + tip="float16 = GPU; int8_float32 = CPU best")) + form.addRow("Language", + _w_line("stt.language", config.stt.language or "", "auto-detect", + tip="Force a language code (e.g. 'de', 'en'). Blank = auto.")) + form.addRow("Beam size", + _w_spin("stt.beam_size", config.stt.beam_size, 1, 10, + tip="1 = fastest greedy; 5 = slightly more accurate")) + form.addRow("Use prev context", + _w_check("stt.condition_on_previous_text", + config.stt.condition_on_previous_text, + tip="Pass prior transcript as context (may improve or hallucinate)")) + body_lay.addWidget(grp) + + # ── TTS ───────────────────────────────────────────────────────── + grp, form = _sec("🔊 Text-to-Speech (TTS)") + form.addRow(f"Enabled{R}", + _w_check("tts.enabled", config.tts.enabled, + tip="Disable for silent / text-only output")) + form.addRow(f"Backend{R}", + _w_combo("tts.backend", ["piper", "f5"], config.tts.backend)) + form.addRow(f"Device{R}", + _w_combo("tts.device", + ["auto", "cpu", "cuda", "cuda:0", "cuda:1", "cuda:2", "cuda:3"], + config.tts.device)) + form.addRow(f"Default voice{R}", + _w_line("tts.default_voice", config.tts.default_voice, + tip="Voice name from the Voices tab")) + form.addRow("Length scale", + _w_dspin("tts.piper_length_scale", config.tts.piper_length_scale, + 0.5, 2.0, 0.05, 2, + tip=">1.0 = slower, <1.0 = faster")) + body_lay.addWidget(grp) + + # ── Voice Training ─────────────────────────────────────────────── + grp, form = _sec("🎓 Voice Training (overnight)") + form.addRow(f"Enabled{L}", + _w_check("training.enabled", config.training.enabled, + tip="Train the default voice automatically each night")) + form.addRow("Start hour (0-23)", + _w_spin("training.start_hour", config.training.start_hour, 0, 23, + tip="Hour the nightly training window opens (3 = 03:00)")) + form.addRow("End hour (0-23)", + _w_spin("training.end_hour", config.training.end_hour, 0, 23, + tip="Hour the window closes (6 = 06:00)")) + form.addRow("Min remaining (min)", + _w_spin("training.min_remaining_minutes", + config.training.min_remaining_minutes, 0, 1440, 15, + tip="Don't start a run if less time of the window remains")) + form.addRow("Max epochs", + _w_spin("training.max_epochs", config.training.max_epochs, 1, 1_000_000, 500)) + form.addRow("Batch size", + _w_spin("training.batch_size", config.training.batch_size, 1, 512)) + form.addRow("Workers", + _w_spin("training.num_workers", config.training.num_workers, 0, 16)) + form.addRow("Accelerator", + _w_combo("training.accelerator", ["gpu", "cpu"], + config.training.accelerator)) + form.addRow("Precision", + _w_combo("training.precision", + ["32", "16-mixed", "bf16-mixed"], + config.training.precision)) + form.addRow("Language", + _w_line("training.language", config.training.language or "de", "", + tip="Piper language code (de = German)")) + body_lay.addWidget(grp) + + # ── Audio ──────────────────────────────────────────────────────── + grp, form = _sec("🎙 Audio") + form.addRow(f"Sample rate{R}", + _w_combo("audio.sample_rate", + [8000, 16000, 22050, 44100, 48000], + config.audio.sample_rate)) + form.addRow(f"Input device{R}", + _w_line("audio.input_device", + "" if config.audio.input_device is None + else str(config.audio.input_device), + "auto (blank = system default)", + tip="Numeric device index from sounddevice.query_devices()")) + form.addRow(f"Output device{R}", + _w_line("audio.output_device", + "" if config.audio.output_device is None + else str(config.audio.output_device), + "auto (blank = system default)", + tip="Numeric device index from sounddevice.query_devices()")) + form.addRow(f"Input gain{L}", + _w_dspin("audio.input_gain", config.audio.input_gain, 0.1, 5.0, 0.1, 1, + tip="Microphone gain multiplier")) + form.addRow(f"Thinking cue delay (s){L}", + _w_dspin("audio.thinking_cue_delay_s", config.audio.thinking_cue_delay_s, + 0.0, 15.0, 0.5, 1, + tip="Soft tick after this many seconds of silence while Javis is still " + "working (repeats every 4 s). 0 disables.")) + body_lay.addWidget(grp) + + # ── Wake Word ──────────────────────────────────────────────────── + grp, form = _sec("👂 Wake Word") + form.addRow(f"Enabled{R}", + _w_check("wake_word.enabled", config.wake_word.enabled)) + form.addRow(f"Model{R}", + _w_line("wake_word.model", config.wake_word.model, + tip="openWakeWord model name, e.g. hey_jarvis")) + + thr_row = QtWidgets.QHBoxLayout() + thr_row.setSpacing(6) + thr_sp = _w_dspin("wake_word.threshold", config.wake_word.threshold, + 0.01, 1.0, 0.05, + tip="Confidence threshold — higher = fewer false wakes") + thr_pct = QtWidgets.QLabel(f"{int(config.wake_word.threshold * 100)} %") + self._th(thr_pct, lambda: f"color: {_CL_MUTED}; font-size: 8.5pt; min-width: 36px; background: transparent;") + def _on_thr(v): + thr_pct.setText(f"{int(v * 100)} %") + if self.orch and self.orch.wake: + self.orch.wake.threshold = v + thr_sp.valueChanged.connect(_on_thr) + thr_row.addWidget(thr_sp) + thr_row.addWidget(thr_pct) + form.addRow(f"Threshold{L}", thr_row) + + form.addRow("VAD threshold", + _w_dspin("wake_word.vad_threshold", config.wake_word.vad_threshold, + 0.01, 1.0, 0.05, + tip="Silero VAD threshold for the capture window")) + form.addRow("Silence (ms)", + _w_spin("wake_word.silence_ms", config.wake_word.silence_ms, + 200, 3000, 100, + tip="Milliseconds of silence that ends an utterance")) + body_lay.addWidget(grp) + + # ── Agent ──────────────────────────────────────────────────────── + grp, form = _sec("🧠 Agent") + + preset_row = QtWidgets.QHBoxLayout() + preset_row.setSpacing(6) + preset_combo = _w_combo("agent.preset", + [ps.key for ps in PRESETS], + config.agent.preset or "default") + for i, ps in enumerate(PRESETS): + preset_combo.setItemText(i, ps.label) + preset_apply = QtWidgets.QPushButton("Apply ⚡") + preset_apply.setFixedWidth(74) + self._th(preset_apply, _btn_sm) + def _apply_preset(): + idx = preset_combo.currentIndex() + if 0 <= idx < len(PRESETS) and self.orch: + ps = PRESETS[idx] + self.orch.conv.system_prompt = ps.text + self._append("error", f"🎭 Persona: {ps.label}") + preset_apply.clicked.connect(_apply_preset) + preset_row.addWidget(preset_combo, 1) + preset_row.addWidget(preset_apply) + form.addRow(f"Persona{L}", preset_row) + + form.addRow("History turns", + _w_spin("agent.history_max_turns", config.agent.history_max_turns, + 5, 100, tip="Max conversation turns kept in LLM context")) + form.addRow("Max iterations", + _w_spin("agent.max_llm_iterations", config.agent.max_llm_iterations, + 1, 20, tip="Max LLM ↔ tool-call cycles per turn")) + form.addRow(f"Spoken error phrase{L}", + _w_line("agent.error_phrase", config.agent.error_phrase, + "empty = silent on errors", + tip="Said aloud when a voice turn fails so you get feedback away from the screen")) + form.addRow(f"Approval phrase{L}", + _w_line("agent.approval_phrase", config.agent.approval_phrase, + "empty = chime only", + tip="Said aloud when a tool call waits for approval and the window isn't focused")) + body_lay.addWidget(grp) + + # ── Memory ─────────────────────────────────────────────────────── + grp, form = _sec("💾 Memory Extraction") + form.addRow("Model", + _w_line("memory.model", config.memory.model or "", + "inherit from llm.model", + tip="Dedicated extraction model. Blank = use llm.model.")) + form.addRow("Temperature", + _w_dspin("memory.temperature", config.memory.temperature, + 0.0, 1.0, 0.05, + tip="Low values (0.1) give more consistent JSON extraction")) + form.addRow("Context (tokens)", + _w_spin("memory.num_ctx", config.memory.num_ctx, 512, 32768, 512)) + form.addRow("Timeout (s)", + _w_spin("memory.timeout_seconds", + config.memory.timeout_seconds + if config.memory.timeout_seconds is not None else 120, + 10, 600, 10)) + body_lay.addWidget(grp) + + # ── Speaker ID ─────────────────────────────────────────────────── + grp, form = _sec("🔊 Speaker Identification") + form.addRow(f"Enabled{R}", + _w_check("speaker_id.enabled", config.speaker_id.enabled, + tip="Requires: pip install speechbrain")) + form.addRow(f"Model{R}", + _w_combo("speaker_id.model", ["speechbrain"], config.speaker_id.model)) + form.addRow(f"Threshold{L}", + _w_dspin("speaker_id.threshold", config.speaker_id.threshold, + 0.30, 0.99, 0.05, + tip="Cosine similarity threshold — 0.75 recommended")) + form.addRow("Min enroll clips", + _w_spin("speaker_id.min_enroll_clips", + config.speaker_id.min_enroll_clips, 1, 10, + tip="Voice clips required to create a profile")) + form.addRow(f"Auto-enroll{L}", + _w_check("speaker_id.auto_enroll", config.speaker_id.auto_enroll, + tip="Ask for name and enroll unknown speakers automatically")) + form.addRow(QtWidgets.QLabel("")) # spacer + form.addRow(QtWidgets.QLabel("Unknown / guest speaker:")) + form.addRow(f" Can respond{L}", + _w_check("speaker_id.unknown_can_respond", + config.speaker_id.unknown_can_respond, + tip="When disabled, Javis stays silent for unrecognised voices")) + form.addRow(" Allowed tools", + _w_line("speaker_id.unknown_allowed_tools", + config.speaker_id.unknown_allowed_tools, + tip="'*' for all tools, or comma-separated fnmatch patterns " + "(e.g. 'spotify__*,homey__lights*')")) + form.addRow(" Denied tools", + _w_line("speaker_id.unknown_denied_tools", + config.speaker_id.unknown_denied_tools, + tip="Comma-separated patterns always blocked " + "(e.g. 'homey__*' to deny all Homey tools for guests)")) + body_lay.addWidget(grp) + + # ── MCP ────────────────────────────────────────────────────────── + grp, form = _sec("🔌 MCP Servers") + + mcp_row = QtWidgets.QHBoxLayout() + mcp_row.setSpacing(6) + mcp_edit = _w_line("mcp.servers_config", config.mcp.servers_config, + tip="Path to mcp_servers.json") + mcp_browse = QtWidgets.QPushButton("Browse…") + mcp_browse.setFixedWidth(74) + self._th(mcp_browse, _btn_sm) + def _browse_mcp(): + p, _ = QtWidgets.QFileDialog.getOpenFileName( + self, "Select MCP config", "", "JSON files (*.json)") + if p: + mcp_edit.setText(p) + mcp_browse.clicked.connect(_browse_mcp) + mcp_row.addWidget(mcp_edit, 1) + mcp_row.addWidget(mcp_browse) + form.addRow(f"Servers config{R}", mcp_row) + + form.addRow("Approval default", + _w_combo("mcp.approval_default", ["ask", "allow", "deny"], + config.mcp.approval_default, + tip="Default tool-call approval policy")) + + mcp_clear = QtWidgets.QPushButton("Clear approval cache ⚡") + self._th(mcp_clear, _btn_sm) + def _clear_mcp(): + if self.orch: + self.orch.mcp.clear_approvals() + self._append("error", "🔐 MCP approval cache cleared.") + mcp_clear.clicked.connect(_clear_mcp) + form.addRow("Approvals", mcp_clear) + + form.addRow("Call timeout (s)", + _w_spin("mcp.call_timeout_seconds", + config.mcp.call_timeout_seconds, 5, 300, 5)) + form.addRow(f"Approval timeout (s){L}", + _w_spin("mcp.approval_timeout_seconds", + config.mcp.approval_timeout_seconds, 0, 600, 5, + tip="Auto-deny a tool call if the approval dialog isn't answered in time " + "so voice turns can't hang. 0 = wait forever.")) + body_lay.addWidget(grp) + + # ── Interface ────────────────────────────────────────────────────────────── + grp, form = _sec("🖥 Interface") + form.addRow("Startup dialog", + _w_check("gui.show_startup_dialog", config.gui.show_startup_dialog, + tip="Show the configuration dialog before the main window on every launch")) + form.addRow(f"Close hides to tray{L}", + _w_check("gui.minimize_to_tray", config.gui.minimize_to_tray, + tip="Closing the window keeps Javis running in the system tray. " + "Quit via File → Quit or the tray menu.")) + form.addRow(f"Global PTT hotkey{R}", + _w_line("gui.global_ptt_hotkey", config.gui.global_ptt_hotkey, + "e.g. ctrl+alt+space (empty = off)", + tip="System-wide push-to-talk shortcut (Windows). Works while the window is " + "hidden or unfocused. Needs at least one modifier.")) + body_lay.addWidget(grp) + + # ── Paths ──────────────────────────────────────────────────────── + grp, form = _sec("📁 Paths") + + base_row = QtWidgets.QHBoxLayout() + base_row.setSpacing(6) + base_edit = _w_line("paths.base_dir", str(config.paths.base_dir), + tip="Javis data directory (logs, voices, models, cache)") + base_browse = QtWidgets.QPushButton("Browse…") + base_browse.setFixedWidth(74) + self._th(base_browse, _btn_sm) + def _browse_base(): + d = QtWidgets.QFileDialog.getExistingDirectory( + self, "Select Javis data directory", str(config.paths.base_dir)) + if d: + base_edit.setText(d) + base_browse.clicked.connect(_browse_base) + base_row.addWidget(base_edit, 1) + base_row.addWidget(base_browse) + form.addRow(f"Data directory{R}", base_row) + form.addRow("Config file", + QtWidgets.QLabel(str(config.source_path or ""))) + body_lay.addWidget(grp) + + # ── legend ─────────────────────────────────────────────────────── + legend = QtWidgets.QLabel(" ↺ requires restart ⚡ applies immediately") + legend.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter) + self._th(legend, lambda: ( + f"color: {_CL_MUTED}; font-size: 8.5pt; padding: 8px 0 4px 0;" + f" background: transparent;")) + body_lay.addWidget(legend) + body_lay.addStretch() + return w + + def _save_config_to_yaml(self): + if config.source_path is None: + QtWidgets.QMessageBox.warning( + self, "No config file", + "Javis is running with built-in defaults — no config.yaml was loaded.\n\n" + "Create a config.yaml next to the Javis launcher and restart." + ) + return + + # ── collect all widget values ──────────────────────────────── + updates: dict[str, dict] = {} + for key, widget in self._cfg_widgets.items(): + section, field = key.split(".", 1) + updates.setdefault(section, {}) + + if isinstance(widget, QtWidgets.QCheckBox): + val: object = widget.isChecked() + elif isinstance(widget, QtWidgets.QSpinBox): + val = widget.value() + elif isinstance(widget, QtWidgets.QDoubleSpinBox): + val = round(widget.value(), 4) + elif isinstance(widget, QtWidgets.QComboBox): + if key == "agent.preset": + from ..agent.prompts import PRESETS as _P + idx = widget.currentIndex() + val = _P[idx].key if 0 <= idx < len(_P) else "default" + elif key == "audio.sample_rate": + val = int(widget.currentText()) + else: + val = widget.currentText() + elif isinstance(widget, QtWidgets.QLineEdit): + text = widget.text().strip() + if key in ("audio.input_device", "audio.output_device"): + val = int(text) if text.lstrip("-").isdigit() else None + elif key in ("stt.language", "memory.model"): + val = text or None + else: + val = text + else: + continue + + updates[section][field] = val + + # ── comment-preserving merge + atomic write ───────────────── + try: + update_config_file(config.source_path, updates) + except Exception as exc: + QtWidgets.QMessageBox.critical(self, "Save failed", str(exc)) + return + + self._cfg_save_lbl.setText("✓ Saved — restart for ↺ changes") + QtCore.QTimer.singleShot(4000, lambda: self._cfg_save_lbl.setText("")) + + # Apply live (⚡) settings immediately. + _mem_upd = updates.get("memory", {}) + if "max_facts" in _mem_upd: + config.memory.max_facts = int(_mem_upd["max_facts"]) + if "max_existing_for_prompt" in _mem_upd: + config.memory.max_existing_for_prompt = int(_mem_upd["max_existing_for_prompt"]) + _gui_upd = updates.get("gui", {}) + if "minimize_to_tray" in _gui_upd: + config.gui.minimize_to_tray = bool(_gui_upd["minimize_to_tray"]) + if "show_startup_dialog" in _gui_upd: + config.gui.show_startup_dialog = bool(_gui_upd["show_startup_dialog"]) + _agent_upd = updates.get("agent", {}) + if "error_phrase" in _agent_upd: + config.agent.error_phrase = str(_agent_upd["error_phrase"]) + if "approval_phrase" in _agent_upd: + config.agent.approval_phrase = str(_agent_upd["approval_phrase"]) + _audio_upd = updates.get("audio", {}) + if "thinking_cue_delay_s" in _audio_upd: + config.audio.thinking_cue_delay_s = float(_audio_upd["thinking_cue_delay_s"]) + _mcp_upd = updates.get("mcp", {}) + if "approval_timeout_seconds" in _mcp_upd: + config.mcp.approval_timeout_seconds = int(_mcp_upd["approval_timeout_seconds"]) + + # Sync model combo if the Settings tab changed llm.model + _settings_model = updates.get("llm", {}).get("model", "") + if _settings_model and _settings_model != self._model_combo.currentText(): + if self._model_combo.findText(_settings_model) == -1: + self._model_combo.insertItem(0, _settings_model) + # blockSignals to avoid double-persisting (already written above) + self._model_combo.blockSignals(True) + self._model_combo.setCurrentText(_settings_model) + self._model_combo.blockSignals(False) + config.llm.model = _settings_model + self._lbl_llm_model.setText(_settings_model) + + + + # ── Intelligence tab ───────────────────────────────────────────────── + + def _build_intelligence_tab(self): + w = QtWidgets.QWidget() + self._th(w, lambda: f"QWidget {{ background: {_CL_BG}; }}") + outer = QtWidgets.QVBoxLayout(w) + outer.setContentsMargins(0, 0, 0, 0) + outer.setSpacing(0) + + toolbar = QtWidgets.QWidget() + toolbar.setFixedHeight(40) + self._th(toolbar, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-bottom: 1px solid {_CL_BORDER}; }}" + )) + tb = QtWidgets.QHBoxLayout(toolbar) + tb.setContentsMargins(12, 0, 12, 0) + tb.setSpacing(6) + lbl_title = QtWidgets.QLabel("Intelligence") + self._th(lbl_title, lambda: ( + f"color: {_CL_TEXT}; font-weight: 600; font-size: 10.5pt; " + f"background: transparent; border: 0;" + )) + tb.addWidget(lbl_title) + tb.addStretch(1) + btn_refresh = QtWidgets.QPushButton("↺ Refresh all") + self._th(btn_refresh, self._ghost_qss) + btn_refresh.setFixedHeight(28) + btn_refresh.clicked.connect(self._refresh_intelligence) + tb.addWidget(btn_refresh) + outer.addWidget(toolbar) + + body = QtWidgets.QWidget() + self._th(body, lambda: f"QWidget {{ background: {_CL_BG}; }}") + body_lay = QtWidgets.QHBoxLayout(body) + body_lay.setContentsMargins(0, 0, 0, 0) + body_lay.setSpacing(0) + + mem_widget, mem_cards_lay, mem_count_lbl = self._build_kanban_column( + "🧠 Memory", lambda: self._refresh_mem_cards(), + add_fn=lambda: self._add_memory_entry() + ) + self._mem_cards_layout = mem_cards_lay + self._mem_count_lbl = mem_count_lbl + + # ── Extraction settings footer (bottom of Memory column) ────── + mem_footer = QtWidgets.QWidget() + self._th(mem_footer, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-top: 1px solid {_CL_BORDER}; }}" + )) + mf_lay = QtWidgets.QGridLayout(mem_footer) + mf_lay.setContentsMargins(12, 8, 12, 8) + mf_lay.setHorizontalSpacing(8) + mf_lay.setVerticalSpacing(4) + + def _muted_lbl(text: str) -> QtWidgets.QLabel: + lbl = QtWidgets.QLabel(text) + self._th(lbl, lambda: ( + f"color: {_CL_MUTED}; font-size: 8pt; " + f"background: transparent; border: 0;" + )) + return lbl + + mf_lay.addWidget(_muted_lbl("Max facts / turn"), 0, 0) + spin_facts = QtWidgets.QSpinBox() + spin_facts.setRange(1, 5) + spin_facts.setValue(getattr(config.memory, "max_facts", 2)) + spin_facts.setFixedWidth(56) + spin_facts.setToolTip( + "Maximum new facts extracted per turn (default: 2). " + "Lower = fewer duplicate smart-home rules." + ) + mf_lay.addWidget(spin_facts, 0, 1) + + mf_lay.addWidget(_muted_lbl("Recall history"), 1, 0) + spin_recall = QtWidgets.QSpinBox() + spin_recall.setRange(5, 50) + spin_recall.setSingleStep(5) + spin_recall.setValue(getattr(config.memory, "max_existing_for_prompt", 25)) + spin_recall.setFixedWidth(56) + spin_recall.setToolTip( + "Existing entries shown to extraction LLM for dedup (default: 25). " + "Raise if near-duplicate entries keep appearing." + ) + mf_lay.addWidget(spin_recall, 1, 1) + + apply_mem_btn = QtWidgets.QPushButton("Apply") + apply_mem_btn.setFixedHeight(26) + self._th(apply_mem_btn, self._ghost_qss) + apply_mem_btn.setToolTip("Save extraction settings to config.yaml") + apply_mem_btn.clicked.connect(self._save_config_to_yaml) + mf_lay.addWidget(apply_mem_btn, 0, 2, 2, 1) + mf_lay.setColumnStretch(1, 1) + + # Register widgets so _save_config_to_yaml picks them up. + self._cfg_widgets["memory.max_facts"] = spin_facts + self._cfg_widgets["memory.max_existing_for_prompt"] = spin_recall + + mem_widget.layout().addWidget(mem_footer) + + qk_widget, qk_cards_lay, qk_count_lbl = self._build_kanban_column( + "⚡ Quick Commands", lambda: self._refresh_quick_cards(), + add_fn=lambda: self._add_quick_command() + ) + self._quick_cards_layout = qk_cards_lay + self._quick_count_lbl = qk_count_lbl + + body_lay.addWidget(mem_widget, 1) + sep = QtWidgets.QFrame() + sep.setFrameShape(QtWidgets.QFrame.Shape.VLine) + self._th(sep, lambda: f"color: {_CL_BORDER};") + body_lay.addWidget(sep) + body_lay.addWidget(qk_widget, 1) + + sep2 = QtWidgets.QFrame() + sep2.setFrameShape(QtWidgets.QFrame.Shape.VLine) + self._th(sep2, lambda: f"color: {_CL_BORDER};") + vp_widget, vp_cards_lay, vp_count_lbl = self._build_kanban_column( + "🎤 Voice Profiles", lambda: self._refresh_voice_cards(), + add_fn=lambda: self._add_voice_profile() + ) + self._voice_cards_layout = vp_cards_lay + self._voice_count_lbl = vp_count_lbl + # Inject a "last identified" sub-label into the Voice Profiles header. + _sp_lbl = QtWidgets.QLabel("") + self._th(_sp_lbl, lambda: ( + f"color: {_CL_MUTED}; font-size: 8pt; background: transparent; border: 0;" + )) + vp_header_widget = vp_widget.layout().itemAt(0).widget() + if vp_header_widget is not None: + vp_header_widget.layout().insertWidget(2, _sp_lbl) + self._current_speaker_lbl = _sp_lbl + body_lay.addWidget(sep2) + body_lay.addWidget(vp_widget, 1) + + outer.addWidget(body, 1) + + self._refresh_mem_cards() + self._refresh_quick_cards() + self._refresh_voice_cards() + return w + + def _build_kanban_column(self, title: str, refresh_fn, add_fn=None) -> tuple: + """Build one Kanban column. Returns (widget, cards_layout, count_label).""" + col = QtWidgets.QWidget() + self._th(col, lambda: f"QWidget {{ background: {_CL_BG}; }}") + cv = QtWidgets.QVBoxLayout(col) + cv.setContentsMargins(0, 0, 0, 0) + cv.setSpacing(0) + + header = QtWidgets.QWidget() + header.setFixedHeight(42) + self._th(header, lambda: ( + f"QWidget {{ background: {_CL_SURFACE}; border-bottom: 1px solid {_CL_BORDER}; }}" + )) + hl = QtWidgets.QHBoxLayout(header) + hl.setContentsMargins(16, 0, 10, 0) + hl.setSpacing(8) + title_lbl = QtWidgets.QLabel(title) + self._th(title_lbl, lambda: ( + f"color: {_CL_TEXT}; font-weight: 600; font-size: 10pt; " + f"background: transparent; border: 0;" + )) + count_lbl = QtWidgets.QLabel("—") + self._th(count_lbl, lambda: ( + f"color: {_CL_MUTED}; font-size: 8.5pt; background: transparent; border: 0;" + )) + btn_ref = QtWidgets.QPushButton("↺") + btn_ref.setFixedSize(26, 26) + btn_ref.setToolTip("Refresh this column") + self._th(btn_ref, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"border-radius: 4px; font-size: 12pt; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; background: {_CL_SURFACE2}; }}" + )) + btn_ref.clicked.connect(refresh_fn) + hl.addWidget(title_lbl) + hl.addWidget(count_lbl, 1) + if add_fn is not None: + btn_add = QtWidgets.QPushButton("+") + btn_add.setFixedSize(26, 26) + btn_add.setToolTip("Add new entry") + self._th(btn_add, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_ACCENT}; " + f"border-radius: 4px; font-size: 14pt; font-weight: 600; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ACCENT}; background: {_CL_SURFACE2}; }}" + )) + btn_add.clicked.connect(add_fn) + hl.addWidget(btn_add) + hl.addWidget(btn_ref) + cv.addWidget(header) + + scroll = QtWidgets.QScrollArea() + scroll.setWidgetResizable(True) + scroll.setFrameShape(QtWidgets.QFrame.Shape.NoFrame) + scroll.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarPolicy.ScrollBarAlwaysOff) + self._th(scroll, lambda: f"QScrollArea {{ background: {_CL_BG}; border: 0; }}") + + inner = QtWidgets.QWidget() + self._th(inner, lambda: f"QWidget {{ background: {_CL_BG}; }}") + cards_lay = QtWidgets.QVBoxLayout(inner) + cards_lay.setContentsMargins(12, 12, 12, 16) + cards_lay.setSpacing(8) + cards_lay.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + + scroll.setWidget(inner) + cv.addWidget(scroll, 1) + return col, cards_lay, count_lbl + + def _refresh_intelligence(self): + self._refresh_mem_cards() + self._refresh_quick_cards() + self._refresh_voice_cards() + + def _refresh_mem_cards(self): + while self._mem_cards_layout.count(): + item = self._mem_cards_layout.takeAt(0) + if item.widget(): + item.widget().deleteLater() + mem_path = config.paths.base_dir / "memory.json" + if not mem_path.exists(): + self._mem_count_lbl.setText("file not found") + return + try: + data = json.loads(mem_path.read_text(encoding="utf-8")) + except Exception as exc: + self._mem_count_lbl.setText(f"error: {exc}") + return + entries = data.get("entries", {}) + ordered = sorted( + entries.items(), + key=lambda kv: (kv[1].get("updated_at", "") if isinstance(kv[1], dict) else ""), + reverse=True, + ) + for key, entry in ordered: + raw = entry if isinstance(entry, dict) else {"value": str(entry), "category": "general"} + self._mem_cards_layout.addWidget(self._make_memory_card(key, raw)) + n = len(ordered) + self._mem_count_lbl.setText(f"{n} entr{'y' if n == 1 else 'ies'}") + + def _refresh_quick_cards(self): + while self._quick_cards_layout.count(): + item = self._quick_cards_layout.takeAt(0) + if item.widget(): + item.widget().deleteLater() + self._quick_card_map.clear() + qk_path = config.paths.quick_commands_file + if not qk_path.exists(): + self._quick_count_lbl.setText("file not found") + return + try: + data = json.loads(qk_path.read_text(encoding="utf-8")) + except Exception as exc: + self._quick_count_lbl.setText(f"error: {exc}") + return + commands = data.get("commands", {}) + ordered = sorted(commands.items(), key=lambda kv: kv[1].get("created_at", "")) + for cmd_id, cmd in ordered: + card = self._make_quick_card(cmd_id, cmd) + self._quick_cards_layout.addWidget(card) + self._quick_card_map[cmd_id] = card + n = len(ordered) + self._quick_count_lbl.setText(f"{n} entr{'y' if n == 1 else 'ies'}") + + _CAT_COLORS: dict = { + "smart_home": "#c07a30", + "spotify": "#1db954", + "user": "#5a8fcf", + "personal": "#7a6a9e", + "general": "#606060", + } + _MODE_COLORS: dict = { + "contains": "#3d9970", + "exact": "#5a8fcf", + "regex": "#e08a3c", + } + + def _make_memory_card(self, key: str, entry: dict) -> QtWidgets.QFrame: + card = QtWidgets.QFrame() + self._th(card, self._card_qss) + v = QtWidgets.QVBoxLayout(card) + v.setContentsMargins(14, 11, 14, 9) + v.setSpacing(5) + + h = QtWidgets.QHBoxLayout() + h.setSpacing(6) + key_lbl = QtWidgets.QLabel(key) + self._th(key_lbl, lambda: f"color: {_CL_TEXT}; font-weight: 600; font-size: 10pt;") + h.addWidget(key_lbl, 1) + category = entry.get("category", "general") + cat_color = self._CAT_COLORS.get(category, "#555") + cat_lbl = QtWidgets.QLabel(f" {category} ") + cat_lbl.setStyleSheet( + f"background: {cat_color}22; color: {cat_color}; font-size: 8pt; " + f"border-radius: 3px; padding: 1px 4px;" + ) + h.addWidget(cat_lbl) + v.addLayout(h) + + value = entry.get("value", "") + if isinstance(value, dict): + value = json.dumps(value, ensure_ascii=False) + val_lbl = QtWidgets.QLabel(str(value)[:400]) + val_lbl.setWordWrap(True) + self._th(val_lbl, lambda: f"color: {_CL_MUTED}; font-size: 9.5pt;") + v.addWidget(val_lbl) + + f_row = QtWidgets.QHBoxLayout() + f_row.setSpacing(4) + ts = (entry.get("updated_at") or entry.get("created_at") or "")[:10] + ts_lbl = QtWidgets.QLabel(ts) + self._th(ts_lbl, lambda: f"color: {_CL_MUTED}; font-size: 8pt;") + del_btn = QtWidgets.QPushButton("✕") + del_btn.setFixedSize(24, 24) + del_btn.setToolTip(f"Delete '{key}'") + self._th(del_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ERROR}; background: {_CL_ERROR_BG}; }}")) + del_btn.clicked.connect(lambda _=False, k=key: self._delete_memory_entry(k)) + edit_btn = QtWidgets.QPushButton("✎") + edit_btn.setFixedSize(24, 24) + edit_btn.setToolTip(f"Edit '{key}'") + self._th(edit_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: #5a8fcf; background: #101a2a; }}")) + edit_btn.clicked.connect(lambda _=False, k=key, e=entry: self._edit_memory_entry(k, e)) + f_row.addWidget(ts_lbl, 1) + f_row.addWidget(edit_btn) + f_row.addWidget(del_btn) + v.addLayout(f_row) + return card + + def _make_quick_card(self, cmd_id: str, cmd: dict) -> QtWidgets.QFrame: + card = QtWidgets.QFrame() + self._th(card, self._card_qss) + v = QtWidgets.QVBoxLayout(card) + v.setContentsMargins(14, 11, 14, 9) + v.setSpacing(5) + + h = QtWidgets.QHBoxLayout() + h.setSpacing(6) + trigger = cmd.get("trigger", "") + trig_lbl = QtWidgets.QLabel(trigger) + self._th(trig_lbl, lambda: f"color: {_CL_TEXT}; font-weight: 600; font-size: 10pt;") + h.addWidget(trig_lbl, 1) + mode = cmd.get("match_mode", "contains") + mode_color = self._MODE_COLORS.get(mode, "#555") + mode_lbl = QtWidgets.QLabel(f" {mode} ") + mode_lbl.setStyleSheet( + f"background: {mode_color}22; color: {mode_color}; font-size: 8pt; " + f"border-radius: 3px; padding: 1px 4px;" + ) + h.addWidget(mode_lbl) + v.addLayout(h) + + actions = cmd.get("actions", []) + lines = [] + for a in actions: + tool = a.get("tool", "?") + args = a.get("args", {}) + args_str = " ".join(f"{k}={v!r}" for k, v in args.items()) if args else "" + lines.append(f"⚡ {tool}") + if args_str: + lines.append(f" {args_str}") + act_lbl = QtWidgets.QLabel("\n".join(lines) if lines else "—") + act_lbl.setWordWrap(True) + self._th(act_lbl, lambda: ( + f"color: {_CL_PURPLE}; font-size: 9pt; " + f"font-family: 'Consolas', 'Courier New', monospace;")) + v.addWidget(act_lbl) + + f_row = QtWidgets.QHBoxLayout() + f_row.setSpacing(4) + ts = (cmd.get("created_at") or "")[:10] + ts_lbl = QtWidgets.QLabel(ts) + self._th(ts_lbl, lambda: f"color: {_CL_MUTED}; font-size: 8pt;") + del_btn = QtWidgets.QPushButton("✕") + del_btn.setFixedSize(24, 24) + del_btn.setToolTip(f"Delete '{trigger}'") + self._th(del_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ERROR}; background: {_CL_ERROR_BG}; }}")) + del_btn.clicked.connect(lambda _=False, i=cmd_id: self._delete_quick_command(i)) + edit_btn = QtWidgets.QPushButton("✎") + edit_btn.setFixedSize(24, 24) + edit_btn.setToolTip(f"Edit '{trigger}'") + self._th(edit_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: #5a8fcf; background: #101a2a; }}")) + edit_btn.clicked.connect(lambda _=False, i=cmd_id, c=cmd: self._edit_quick_command(i, c)) + f_row.addWidget(ts_lbl, 1) + f_row.addWidget(edit_btn) + f_row.addWidget(del_btn) + v.addLayout(f_row) + return card + + # ─── voice-profile panel ───────────────────────────────────────────── + + def _refresh_voice_cards(self): + if self._voice_cards_layout is None: + return + while self._voice_cards_layout.count(): + item = self._voice_cards_layout.takeAt(0) + if item.widget(): + item.widget().deleteLater() + orch = self.orch + profiles: list[dict] = [] + if orch is not None and orch.profiler is not None: + try: + profiles = orch.profiler.list_profiles() + except Exception: + log.debug("[voice] failed to list voice profiles") + if self._voice_count_lbl is not None: + self._voice_count_lbl.setText( + f"{len(profiles)} profile{'s' if len(profiles) != 1 else ''}" + ) + if not profiles: + empty = QtWidgets.QLabel("No voice profiles yet.\nUse + to enroll a speaker.") + empty.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter) + self._th(empty, lambda: f"color: {_CL_MUTED}; font-size: 9pt; padding: 24px;") + self._voice_cards_layout.addWidget(empty) + return + for profile in profiles: + self._voice_cards_layout.addWidget(self._make_voice_card(profile)) + + def _make_voice_card(self, profile: dict) -> QtWidgets.QFrame: + card = QtWidgets.QFrame() + self._th(card, self._card_qss) + clip_count = profile.get("clip_count", 1) + enrolled_at = (profile.get("enrolled_at") or "")[:10] + + h = QtWidgets.QHBoxLayout() + h.setSpacing(6) + name_lbl = QtWidgets.QLabel(f"🎙 {name}") + self._th(name_lbl, lambda: f"color: {_CL_TEXT}; font-weight: 600; font-size: 10pt;") + h.addWidget(name_lbl, 1) + clips_lbl = QtWidgets.QLabel(f" {clip_count} clip{'s' if clip_count != 1 else ''} ") + self._th(clips_lbl, lambda: ( + f"background: {_CL_ACCENT}22; color: {_CL_ACCENT}; font-size: 8pt; " + f"border-radius: 3px; padding: 1px 4px;")) + h.addWidget(clips_lbl) + v.addLayout(h) + + f_row = QtWidgets.QHBoxLayout() + f_row.setSpacing(4) + date_lbl = QtWidgets.QLabel(enrolled_at) + self._th(date_lbl, lambda: f"color: {_CL_MUTED}; font-size: 8pt;") + del_btn = QtWidgets.QPushButton("✕") + del_btn.setFixedSize(24, 24) + del_btn.setToolTip(f"Delete profile for '{name}'") + self._th(del_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ERROR}; background: {_CL_ERROR_BG}; }}")) + del_btn.clicked.connect(lambda _=False, n=name: self._delete_voice_profile(n)) + + priv_btn = QtWidgets.QPushButton("🔐") + priv_btn.setFixedSize(24, 24) + priv_btn.setToolTip(f"Edit access privileges for '{name}'") + self._th(priv_btn, lambda: ( + f"QPushButton {{ background: transparent; border: 0; color: {_CL_MUTED}; " + f"font-size: 10pt; border-radius: 4px; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ACCENT}; background: #2a1810; }}")) + priv_btn.clicked.connect(lambda _=False, n=name: self._edit_voice_privileges(n)) + + f_row.addWidget(date_lbl, 1) + f_row.addWidget(priv_btn) + f_row.addWidget(del_btn) + v.addLayout(f_row) + return card + + def _add_voice_profile(self): + self._enroll_speaker_dialog(None) + + def _delete_voice_profile(self, name: str): + orch = self.orch + if orch is None or orch.profiler is None: + return + reply = QtWidgets.QMessageBox.question( + self, "Delete Profile", + f"Delete voice profile for '{name}'?", + QtWidgets.QMessageBox.StandardButton.Yes | QtWidgets.QMessageBox.StandardButton.No, + ) + if reply == QtWidgets.QMessageBox.StandardButton.Yes: + orch.profiler.delete_profile(name) + self._refresh_voice_cards() + + def _edit_voice_privileges(self, name: str): + """Open a dialog for editing a speaker's access privileges.""" + from ..stt.voice_profiler import SpeakerPrivileges + + orch = self.orch + if orch is None or orch.profiler is None: + return + privs = orch.profiler.get_privileges(name) + + dlg = QtWidgets.QDialog(self) + dlg.setWindowTitle(f"Privileges — {name}") + dlg.setMinimumWidth(400) + dlg.setStyleSheet( + f"QDialog {{ background: {_CL_SURFACE}; }}" + f"QLabel {{ color: {_CL_TEXT}; background: transparent; }}" + f"QLineEdit {{ background: {_CL_BG}; color: {_CL_TEXT}; " + f"border: 1px solid {_CL_BORDER}; border-radius: 6px; padding: 5px 10px; }}" + f"QCheckBox {{ color: {_CL_TEXT}; }}" + ) + lay = QtWidgets.QVBoxLayout(dlg) + lay.setSpacing(10) + lay.setContentsMargins(20, 16, 20, 16) + + lay.addWidget(QtWidgets.QLabel(f"Access privileges for {name}")) + + form = QtWidgets.QFormLayout() + form.setLabelAlignment(QtCore.Qt.AlignmentFlag.AlignRight) + form.setSpacing(8) + + can_resp_chk = QtWidgets.QCheckBox() + can_resp_chk.setChecked(privs.can_respond) + can_resp_chk.setToolTip("Uncheck to have Javis stay completely silent for this speaker") + form.addRow("Allow response:", can_resp_chk) + + # allowed_tools: join list to comma-separated string for editing + if isinstance(privs.allowed_tools, list): + allowed_str = ", ".join(privs.allowed_tools) + else: + allowed_str = privs.allowed_tools # "*" + allowed_edit = QtWidgets.QLineEdit(allowed_str) + allowed_edit.setPlaceholderText("*") + allowed_edit.setToolTip( + "'*' grants all tools; or a comma-separated list of fnmatch patterns " + "(e.g. 'spotify__*, homey__lights*')" + ) + form.addRow("Allowed tools:", allowed_edit) + + denied_edit = QtWidgets.QLineEdit(", ".join(privs.denied_tools)) + denied_edit.setPlaceholderText("e.g. homey__*, spotify__*") + denied_edit.setToolTip( + "Comma-separated fnmatch patterns that are ALWAYS blocked, " + "regardless of the allowed list" + ) + form.addRow("Denied tools:", denied_edit) + + lay.addLayout(form) + + hint = QtWidgets.QLabel( + "Examples: homey__* blocks all Homey tools; " + "spotify__* blocks Spotify." + ) + hint.setWordWrap(True) + hint.setStyleSheet(f"color: {_CL_MUTED}; font-size: 8pt;") + lay.addWidget(hint) + + btn_row = QtWidgets.QHBoxLayout() + save_btn = QtWidgets.QPushButton("Save") + save_btn.setStyleSheet( + f"QPushButton {{ background: {_CL_ACCENT}; color: #fff; border: 0; " + f"border-radius: 8px; padding: 6px 20px; font-weight: 600; }}" + f"QPushButton:hover {{ background: #e0886a; }}" + ) + cancel_btn = QtWidgets.QPushButton("Cancel") + cancel_btn.setStyleSheet( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_MUTED}; border-radius: 8px; padding: 6px 14px; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; border-color: {_CL_TEXT}; }}" + ) + btn_row.addWidget(save_btn, 1) + btn_row.addWidget(cancel_btn) + lay.addLayout(btn_row) + + def _save(): + raw_allowed = allowed_edit.text().strip() + allowed: object = ( + "*" if raw_allowed == "*" + else [p.strip() for p in raw_allowed.split(",") if p.strip()] + ) + denied = [p.strip() for p in denied_edit.text().split(",") if p.strip()] + new_privs = SpeakerPrivileges( + can_respond=can_resp_chk.isChecked(), + allowed_tools=allowed, + denied_tools=denied, + ) + orch.profiler.set_privileges(name, new_privs) + dlg.accept() + + save_btn.clicked.connect(_save) + cancel_btn.clicked.connect(dlg.reject) + dlg.exec() + + def _enroll_speaker_dialog(self, name_prefill: str | None = None): + """Open the speaker-enrollment dialog.""" + dlg = QtWidgets.QDialog(self) + dlg.setWindowTitle("Enroll Speaker") + dlg.setMinimumWidth(380) + dlg.setStyleSheet( + f"QDialog {{ background: {_CL_SURFACE}; }}" + f"QLabel {{ color: {_CL_TEXT}; background: transparent; }}" + f"QLineEdit {{ background: {_CL_BG}; color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 6px; padding: 5px 10px; }}" + ) + lay = QtWidgets.QVBoxLayout(dlg) + lay.setSpacing(12) + lay.setContentsMargins(20, 20, 20, 20) + + lay.addWidget(QtWidgets.QLabel("Speaker name:")) + name_edit = QtWidgets.QLineEdit() + if name_prefill: + name_edit.setText(name_prefill) + lay.addWidget(name_edit) + + status_lbl = QtWidgets.QLabel("Press Start to begin recording.") + status_lbl.setWordWrap(True) + status_lbl.setStyleSheet(f"color: {_CL_MUTED}; font-size: 9pt;") + lay.addWidget(status_lbl) + + btn_row = QtWidgets.QHBoxLayout() + start_btn = QtWidgets.QPushButton("▶ Start Recording") + start_btn.setStyleSheet( + f"QPushButton {{ background: {_CL_ACCENT}; color: #fff; border: 0; " + f"border-radius: 8px; padding: 6px 18px; font-weight: 600; }}" + f"QPushButton:hover {{ background: {_CL_ACCENT2}; }}" + f"QPushButton:disabled {{ background: {_CL_INPUT}; color: {_CL_MUTED}; }}" + ) + cancel_btn = QtWidgets.QPushButton("Cancel") + cancel_btn.setStyleSheet( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_MUTED}; border-radius: 8px; padding: 6px 14px; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; border-color: {_CL_TEXT}; }}" + ) + btn_row.addWidget(start_btn, 1) + btn_row.addWidget(cancel_btn) + lay.addLayout(btn_row) + + def _start(): + nm = name_edit.text().strip() + if not nm: + status_lbl.setText("⚠ Please enter a name first.") + return + start_btn.setEnabled(False) + name_edit.setEnabled(False) + status_lbl.setText("Preparing…") + orch = self.orch + if orch is not None: + asyncio.ensure_future(orch.start_enrollment(nm)) + self._enroll_dlg = dlg + + start_btn.clicked.connect(_start) + cancel_btn.clicked.connect(dlg.reject) + dlg.finished.connect(lambda _: setattr(self, "_enroll_dlg", None)) + dlg.exec() + + def _show_ask_name_dialog(self, payload: dict): + """Non-blocking dialog: enroll as new speaker or add clip to existing one.""" + key = payload.get("key", "") + + existing = [] + if self.orch is not None: + existing = [p["name"] for p in self.orch.profiler.list_profiles()] + + dlg = QtWidgets.QDialog(self) + dlg.setWindowTitle("Unknown Speaker Detected") + dlg.setMinimumWidth(380) + dlg.setStyleSheet( + f"QDialog {{ background: {_CL_SURFACE}; }}" + f"QLabel {{ color: {_CL_TEXT}; background: transparent; }}" + f"QLineEdit {{ background: {_CL_BG}; color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 6px; padding: 5px 10px; }}" + f"QComboBox {{ background: {_CL_BG}; color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 6px; padding: 4px 10px; }}" + f"QComboBox QAbstractItemView {{ background: {_CL_BG}; color: {_CL_TEXT}; " + f"selection-background-color: {_CL_ACCENT}; }}" + f"QRadioButton {{ color: {_CL_TEXT}; background: transparent; spacing: 6px; }}" + ) + lay = QtWidgets.QVBoxLayout(dlg) + lay.setSpacing(14) + lay.setContentsMargins(20, 20, 20, 20) + + lay.addWidget(QtWidgets.QLabel( + "Javis detected an unknown speaker.\nHow would you like to continue?" + )) + + # ── New profile row ─────────────────────────────────────────────── + radio_new = QtWidgets.QRadioButton("New profile:") + radio_new.setChecked(True) + name_edit = QtWidgets.QLineEdit() + name_edit.setPlaceholderText("Enter name…") + row_new = QtWidgets.QHBoxLayout() + row_new.setSpacing(8) + row_new.addWidget(radio_new) + row_new.addWidget(name_edit, 1) + lay.addLayout(row_new) + + # ── Existing profile row ────────────────────────────────────────── + radio_existing = QtWidgets.QRadioButton("Add to existing:") + combo = QtWidgets.QComboBox() + for nm in existing: + combo.addItem(nm) + combo.setEnabled(bool(existing)) + radio_existing.setEnabled(bool(existing)) + row_ex = QtWidgets.QHBoxLayout() + row_ex.setSpacing(8) + row_ex.addWidget(radio_existing) + row_ex.addWidget(combo, 1) + lay.addLayout(row_ex) + + # ── toggle enable/disable ───────────────────────────────────────── + def _update_fields(): + use_new = radio_new.isChecked() + name_edit.setEnabled(use_new) + combo.setEnabled(not use_new and bool(existing)) + if use_new: + name_edit.setFocus() + else: + combo.setFocus() + + radio_new.toggled.connect(_update_fields) + _update_fields() + + # ── buttons ─────────────────────────────────────────────────────── + btn_row = QtWidgets.QHBoxLayout() + enroll_btn = QtWidgets.QPushButton("Enroll") + enroll_btn.setStyleSheet( + f"QPushButton {{ background: {_CL_ACCENT}; color: #fff; border: 0; " + f"border-radius: 8px; padding: 6px 18px; font-weight: 600; }}" + f"QPushButton:hover {{ background: #e0886a; }}" + ) + skip_btn = QtWidgets.QPushButton("Skip") + skip_btn.setStyleSheet( + f"QPushButton {{ background: transparent; border: 1px solid {_CL_BORDER}; " + f"color: {_CL_MUTED}; border-radius: 8px; padding: 6px 14px; }}" + f"QPushButton:hover {{ color: {_CL_TEXT}; }}" + ) + btn_row.addWidget(enroll_btn, 1) + btn_row.addWidget(skip_btn) + lay.addLayout(btn_row) + + def _enroll(): + if radio_new.isChecked(): + nm = name_edit.text().strip() + else: + nm = combo.currentText().strip() + if not nm: + return + orch = self.orch + if orch is not None: + # Use call_soon to defer task creation to the next event-loop + # iteration, avoiding a qasync re-entrancy crash when the + # button click fires while another asyncio task is executing. + asyncio.get_event_loop().call_soon( + lambda: asyncio.ensure_future(orch.auto_enroll_confirm(key, nm)) + ) + dlg.accept() + + def _on_finished(): + self._name_dialogs.pop(key, None) + + enroll_btn.clicked.connect(_enroll) + skip_btn.clicked.connect(dlg.reject) + name_edit.returnPressed.connect(_enroll) + dlg.finished.connect(_on_finished) + self._name_dialogs[key] = dlg + dlg.show() # non-modal — does not block the asyncio event loop + + + def _on_speaker_identified(self, payload: dict): + if self._current_speaker_lbl is None: + return + name = payload.get("name") + if name: + conf = payload.get("confidence", 0.0) + self._current_speaker_lbl.setText(f" Last: {name} ({conf:.0%})") + self._current_speaker_lbl.setStyleSheet( + f"color: {_CL_ACCENT}; font-size: 8pt; background: transparent; border: 0;" + ) + else: + self._current_speaker_lbl.setText(" Last: unknown") + self._current_speaker_lbl.setStyleSheet( + f"color: {_CL_MUTED}; font-size: 8pt; background: transparent; border: 0;" + ) + + def _on_speaker_enrolled(self, payload: dict): + name = payload.get("name", "") + if self._enroll_dlg is not None: + self._enroll_dlg.accept() + self._refresh_voice_cards() + if name: + # defer so the asyncio step that triggered this signal unwinds first + QtCore.QTimer.singleShot( + 0, lambda: QtWidgets.QMessageBox.information( + self, "Enrolled", f"Voice profile for '{name}' saved successfully." + ) + ) + + def _on_enroll_progress(self, payload: dict): + dlg = self._enroll_dlg + if dlg is None: + return + # Find the status label in the dialog. + for lbl in dlg.findChildren(QtWidgets.QLabel): + if lbl.wordWrap(): + error = payload.get("error") + if error: + lbl.setText(f"⚠ {error}") + lbl.setStyleSheet(f"color: {_CL_ERROR}; font-size: 9pt;") + # Re-enable start button on error. + for btn in dlg.findChildren(QtWidgets.QPushButton): + if "Start" in btn.text(): + btn.setEnabled(True) + else: + clip = payload.get("clip", 0) + total = payload.get("total", 0) + recording = payload.get("recording", False) + if recording: + lbl.setText(f"🎙 Recording clip {clip + 1} of {total}…") + else: + lbl.setText(f"✓ Clip {clip} of {total} recorded.") + lbl.setStyleSheet(f"color: {_CL_MUTED}; font-size: 9pt;") + break + + def _delete_memory_entry(self, key: str): + mem_path = config.paths.base_dir / "memory.json" + try: + data = json.loads(mem_path.read_text(encoding="utf-8")) + entries = data.get("entries", {}) + if key in entries: + del entries[key] + data["entries"] = entries + tmp = mem_path.with_suffix(".json.part") + tmp.write_text( + json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8" + ) + tmp.replace(mem_path) + except Exception as exc: + log.warning(f"[GUI] delete memory entry {key!r}: {exc}") + self._refresh_mem_cards() + + def _delete_quick_command(self, cmd_id: str): + if self.orch and self.orch.quick: + self.orch.quick.delete(cmd_id) + else: + qk_path = config.paths.quick_commands_file + try: + data = json.loads(qk_path.read_text(encoding="utf-8")) + data.get("commands", {}).pop(cmd_id, None) + tmp = qk_path.with_suffix(".json.part") + tmp.write_text( + json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8" + ) + tmp.replace(qk_path) + except Exception as exc: + log.warning(f"[GUI] delete quick command {cmd_id!r}: {exc}") + self._refresh_quick_cards() + + def _dialog_qss(self) -> str: + return ( + f"QDialog {{ background: {_CL_BG}; }}" + f"QLabel {{ color: {_CL_TEXT}; background: transparent; }}" + f"QLineEdit, QTextEdit, QPlainTextEdit, QComboBox {{" + f" background: {_CL_SURFACE2}; color: {_CL_TEXT}; " + f" border: 1px solid {_CL_BORDER}; border-radius: 5px; padding: 4px 8px; }}" + f"QLineEdit:focus, QTextEdit:focus {{ border: 1px solid {_CL_ACCENT}; }}" + f"QComboBox {{ padding-right: 8px; }}" + f"QComboBox::drop-down {{ border: 0; }}" + f"QComboBox QAbstractItemView {{" + f" background: {_CL_SURFACE}; color: {_CL_TEXT}; " + f" border: 1px solid {_CL_BORDER}; " + f" selection-background-color: {_CL_ACCENT}; }}" + f"QPushButton {{" + f" background: {_CL_SURFACE}; color: {_CL_TEXT}; " + f" border: 1px solid {_CL_BORDER}; border-radius: 6px; padding: 5px 16px; }}" + f"QPushButton:hover {{ background: {_CL_SURFACE2}; }}" + ) + + def _add_memory_entry(self): + try: + if self._show_memory_dialog(): + self._refresh_mem_cards() + except Exception as exc: + import traceback + log.error(f"[GUI] _add_memory_entry: {exc}\n{traceback.format_exc()}") + QtWidgets.QMessageBox.critical(self, "Error", f"Could not open dialog:\n{exc}") + + def _edit_memory_entry(self, key: str, entry: dict): + try: + if self._show_memory_dialog(key, entry): + self._refresh_mem_cards() + except Exception as exc: + import traceback + log.error(f"[GUI] _edit_memory_entry: {exc}\n{traceback.format_exc()}") + QtWidgets.QMessageBox.critical(self, "Error", f"Could not open dialog:\n{exc}") + + def _show_memory_dialog(self, key: str = "", entry: dict | None = None) -> bool: + """Show add/edit dialog for a memory entry. Returns True if saved.""" + from PySide6.QtWidgets import ( + QDialog, QVBoxLayout, QFormLayout, QHBoxLayout, + QLineEdit, QTextEdit, QComboBox, QPushButton, + ) + from datetime import datetime, timezone + + log.debug(f"[GUI] _show_memory_dialog key={key!r} is_edit={entry is not None}") + is_edit = entry is not None + dlg = QDialog(self) + dlg.setWindowTitle("Edit Memory Entry" if is_edit else "Add Memory Entry") + dlg.setMinimumWidth(480) + dlg.setWindowModality(QtCore.Qt.WindowModality.ApplicationModal) + dlg.setStyleSheet(self._dialog_qss()) + + outer = QVBoxLayout(dlg) + outer.setContentsMargins(20, 16, 20, 16) + outer.setSpacing(10) + + form = QFormLayout() + form.setSpacing(8) + form.setLabelAlignment(QtCore.Qt.AlignmentFlag.AlignRight) + + key_edit = QLineEdit(key) + key_edit.setPlaceholderText("e.g. homey_esszimmer_preference") + + raw_val = entry.get("value", "") if entry else "" + if isinstance(raw_val, dict): + raw_val = json.dumps(raw_val, ensure_ascii=False, indent=2) + val_edit = QTextEdit() + val_edit.setPlainText(str(raw_val)) + val_edit.setMinimumHeight(90) + val_edit.setMaximumHeight(200) + + cats = ["general", "smart_home", "spotify", "user", "personal"] + cat_combo = QComboBox() + cat_combo.addItems(cats) + cur_cat = (entry.get("category", "general") if entry else "general") + cat_combo.setCurrentText(cur_cat if cur_cat in cats else "general") + + form.addRow("Key:", key_edit) + form.addRow("Value:", val_edit) + form.addRow("Category:", cat_combo) + outer.addLayout(form) + + btn_row = QHBoxLayout() + btn_row.setSpacing(8) + btn_row.addStretch(1) + cancel_btn = QPushButton("Cancel") + cancel_btn.clicked.connect(dlg.reject) + save_btn = QPushButton("Save") + save_btn.setStyleSheet( + f"QPushButton {{ background: {_CL_ACCENT}; border: 0; color: white; " + f"font-weight: 600; border-radius: 6px; padding: 5px 18px; }}" + f"QPushButton:hover {{ background: #c06040; }}" + ) + save_btn.clicked.connect(dlg.accept) + save_btn.setDefault(True) + btn_row.addWidget(cancel_btn) + btn_row.addWidget(save_btn) + outer.addLayout(btn_row) + + if dlg.exec() != QDialog.DialogCode.Accepted: + return False + + new_key = key_edit.text().strip() + new_val = val_edit.toPlainText().strip() + new_cat = cat_combo.currentText() + if not new_key or not new_val: + return False + + now = datetime.now(timezone.utc).isoformat() + mem_path = config.paths.base_dir / "memory.json" + try: + data = json.loads(mem_path.read_text(encoding="utf-8")) if mem_path.exists() else {} + except Exception: + log.debug("[memory] failed to read memory.json for key-value update") + data = {} + entries = data.get("entries", {}) + if is_edit and new_key != key and key in entries: + del entries[key] + old = entries.get(new_key) if isinstance(entries.get(new_key), dict) else {} + entries[new_key] = { + "value": new_val, + "category": new_cat, + "created_at": old.get("created_at", now), + "updated_at": now, + } + data["entries"] = entries + tmp = mem_path.with_suffix(".json.part") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + tmp.replace(mem_path) + return True + + def _add_quick_command(self): + try: + if self._show_quick_dialog(): + self._refresh_quick_cards() + except Exception as exc: + import traceback + log.error(f"[GUI] _add_quick_command: {exc}\n{traceback.format_exc()}") + QtWidgets.QMessageBox.critical(self, "Error", f"Could not open dialog:\n{exc}") + + def _edit_quick_command(self, cmd_id: str, cmd: dict): + try: + if self._show_quick_dialog(cmd_id, cmd): + self._refresh_quick_cards() + except Exception as exc: + import traceback + log.error(f"[GUI] _edit_quick_command: {exc}\n{traceback.format_exc()}") + QtWidgets.QMessageBox.critical(self, "Error", f"Could not open dialog:\n{exc}") + + def _show_quick_dialog(self, cmd_id: str = "", cmd: dict | None = None) -> bool: + """Show add/edit dialog for a quick command. Returns True if saved.""" + from PySide6.QtWidgets import ( + QDialog, QVBoxLayout, QFormLayout, QHBoxLayout, + QLineEdit, QTextEdit, QComboBox, QPushButton, QLabel, + ) + import uuid + from datetime import datetime, timezone + + log.debug(f"[GUI] _show_quick_dialog cmd_id={cmd_id!r} is_edit={cmd is not None}") + is_edit = cmd is not None + dlg = QDialog(self) + dlg.setWindowTitle("Edit Quick Command" if is_edit else "Add Quick Command") + dlg.setMinimumWidth(520) + dlg.setWindowModality(QtCore.Qt.WindowModality.ApplicationModal) + dlg.setStyleSheet(self._dialog_qss()) + + outer = QVBoxLayout(dlg) + outer.setContentsMargins(20, 16, 20, 16) + outer.setSpacing(10) + + form = QFormLayout() + form.setSpacing(8) + form.setLabelAlignment(QtCore.Qt.AlignmentFlag.AlignRight) + + trigger_edit = QLineEdit(cmd.get("trigger", "") if cmd else "") + trigger_edit.setPlaceholderText("e.g. esszimmerlicht an") + + modes = ["contains", "exact", "regex"] + mode_combo = QComboBox() + mode_combo.addItems(modes) + cur_mode = cmd.get("match_mode", "contains") if cmd else "contains" + mode_combo.setCurrentText(cur_mode if cur_mode in modes else "contains") + + actions = cmd.get("actions", []) if cmd else [] + act_edit = QTextEdit() + act_edit.setPlainText(json.dumps(actions, ensure_ascii=False, indent=2)) + act_edit.setMinimumHeight(140) + act_edit.setMaximumHeight(300) + + hint = QLabel('JSON array of {"tool": "...", "args": {...}} objects') + hint.setStyleSheet(f"color: {_CL_MUTED}; font-size: 8pt;") + + form.addRow("Trigger:", trigger_edit) + form.addRow("Match mode:", mode_combo) + form.addRow("Actions:", act_edit) + form.addRow("", hint) + outer.addLayout(form) + + btn_row = QHBoxLayout() + btn_row.setSpacing(8) + btn_row.addStretch(1) + cancel_btn = QPushButton("Cancel") + cancel_btn.clicked.connect(dlg.reject) + save_btn = QPushButton("Save") + save_btn.setStyleSheet( + f"QPushButton {{ background: {_CL_ACCENT}; border: 0; color: white; " + f"font-weight: 600; border-radius: 6px; padding: 5px 18px; }}" + f"QPushButton:hover {{ background: #c06040; }}" + ) + save_btn.clicked.connect(dlg.accept) + save_btn.setDefault(True) + btn_row.addWidget(cancel_btn) + btn_row.addWidget(save_btn) + outer.addLayout(btn_row) + + if dlg.exec() != QDialog.DialogCode.Accepted: + return False + + new_trigger = trigger_edit.text().strip() + new_mode = mode_combo.currentText() + actions_text = act_edit.toPlainText().strip() + if not new_trigger or not actions_text: + return False + + try: + new_actions = json.loads(actions_text) + if not isinstance(new_actions, list): + new_actions = [new_actions] + except Exception as exc: + log.warning(f"[GUI] invalid actions JSON: {exc}") + return False + + if is_edit and cmd_id and self.orch and self.orch.quick: + self.orch.quick.update(cmd_id, trigger=new_trigger, match_mode=new_mode, actions=new_actions) + elif not is_edit and self.orch and self.orch.quick: + self.orch.quick.add(new_trigger, new_mode, new_actions) + else: + # Fallback: write JSON directly (orch not yet started) + now = datetime.now(timezone.utc).isoformat() + qk_path = config.paths.quick_commands_file + try: + data = json.loads(qk_path.read_text(encoding="utf-8")) if qk_path.exists() else {} + except Exception: + data = {} + commands = data.get("commands", {}) + new_id = cmd_id if (is_edit and cmd_id) else str(uuid.uuid4()) + old_cmd = commands.get(new_id) if isinstance(commands.get(new_id), dict) else {} + commands[new_id] = { + "id": new_id, + "trigger": new_trigger, + "match_mode": new_mode, + "actions": new_actions, + "created_at": old_cmd.get("created_at", now), + "updated_at": now, + } + data["commands"] = commands + tmp = qk_path.with_suffix(".json.part") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + tmp.replace(qk_path) + return True + + def _on_quick_match(self, p: dict): + cmd_id = p.get("id", "") + card = self._quick_card_map.get(cmd_id) + if card is None: + return + _hi = ( + f"QFrame {{ background: {_CL_SURFACE}; border: 2px solid {_CL_ACCENT}; " + f"border-radius: 10px; }}" + f"QLabel {{ border: 0; background: transparent; }}" + f"QPushButton {{ border: 0; background: transparent; }}" + ) + _norm = ( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_BORDER}; " + f"border-radius: 10px; }}" + f"QLabel {{ border: 0; background: transparent; }}" + f"QPushButton {{ border: 0; background: transparent; }}" + ) + card.setStyleSheet(_hi) + QtCore.QTimer.singleShot(1800, lambda: card.setStyleSheet(_norm) if card else None) + + @staticmethod + def _badge_style(state: str) -> str: + color = _STATE_COLORS.get(state, _CL_MUTED) + return (f"background:{color}; color:white; font-weight:600; " + f"padding:3px 10px; border-radius:5px; font-size:9pt;") + + def _dot_style(self, state: str) -> str: + color = _STATE_COLORS.get(state, _CL_MUTED) + return (f"background:{color}; border-radius:5px;") + + def _set_pulse(self, on: bool): + """Start/stop the status-dot opacity pulse (listening/speaking).""" + if on: + if self._dot_anim is None: + self._dot_fx = QtWidgets.QGraphicsOpacityEffect(self) + self.state_dot.setGraphicsEffect(self._dot_fx) + # Qt 6 removed the "Alternate" animation direction, so the + # ping-pong pulse is a looping group of fade-out + fade-in. + self._dot_anim = QtCore.QSequentialAnimationGroup(self) + fade_out = QtCore.QPropertyAnimation( + self._dot_fx, b"opacity", self) + fade_out.setDuration(700) + fade_out.setStartValue(1.0) + fade_out.setEndValue(0.3) + fade_out.setEasingCurve(QtCore.QEasingCurve.InOutSine) + fade_in = QtCore.QPropertyAnimation( + self._dot_fx, b"opacity", self) + fade_in.setDuration(700) + fade_in.setStartValue(0.3) + fade_in.setEndValue(1.0) + fade_in.setEasingCurve(QtCore.QEasingCurve.InOutSine) + self._dot_anim.addAnimation(fade_out) + self._dot_anim.addAnimation(fade_in) + self._dot_anim.setLoopCount(-1) + self._dot_anim.stop() + self._dot_anim.start() + elif self._dot_anim is not None: + self._dot_anim.stop() + if getattr(self, "_dot_fx", None) is not None: + self._dot_fx.setOpacity(1.0) + + def _build_status_bar(self): + sb = self.statusBar() + # Small round "activity dot" that pulses while listening/speaking. + self.state_dot = QtWidgets.QLabel() + self.state_dot.setFixedSize(10, 10) + self.state_dot.setStyleSheet(self._dot_style("idle")) + sb.addWidget(self.state_dot) + self.state_badge = QtWidgets.QLabel(" idle ") + self.state_badge.setStyleSheet(self._badge_style("idle")) + sb.addWidget(self.state_badge) + # Lazily created in _set_pulse when a pulse is needed. + self._dot_anim: QtCore.QSequentialAnimationGroup | None = None + self._dot_fx: "QtWidgets.QGraphicsOpacityEffect | None" = None + self.sb_phase = QtWidgets.QLabel("starting…") + sb.addWidget(self.sb_phase, 1) + self.sb_mic = QtWidgets.QProgressBar() + self.sb_mic.setRange(0, 100); self.sb_mic.setValue(0) + self.sb_mic.setFixedWidth(120); self.sb_mic.setTextVisible(False) + self.sb_mic.setToolTip("Mic level") + sb.addPermanentWidget(QtWidgets.QLabel("mic")) + sb.addPermanentWidget(self.sb_mic) + self.sb_wake = QtWidgets.QProgressBar() + self.sb_wake.setRange(0, 100); self.sb_wake.setValue(0) + self.sb_wake.setFixedWidth(120); self.sb_wake.setFormat("wake %v%") + self.sb_wake.setToolTip("Wake-word score") + sb.addPermanentWidget(self.sb_wake) + self.sb_tasks = QtWidgets.QLabel("tasks: 0") + sb.addPermanentWidget(self.sb_tasks) + + def _build_menus(self): + menubar = self.menuBar() + file_menu = menubar.addMenu("&File") + act_hide = QtGui.QAction("Hide to tray", self) + act_hide.setShortcut(QtGui.QKeySequence("Ctrl+H")) + act_hide.triggered.connect(self._hide_to_tray) + file_menu.addAction(act_hide) + file_menu.addSeparator() + act_quit = QtGui.QAction("Quit Javis", self) + act_quit.setShortcut(QtGui.QKeySequence("Ctrl+Q")) + act_quit.triggered.connect(self._quit) + file_menu.addAction(act_quit) + + tools = menubar.addMenu("&Tools") + act_models = QtGui.QAction("Startup configuration…", self) + act_models.setToolTip("Reopen the configuration dialog shown at launch (models, devices, persona)") + act_models.triggered.connect(self._manage_models) + tools.addAction(act_models) + tools.addSeparator() + act_theme = QtGui.QAction("Toggle light/dark theme", self) + act_theme.setCheckable(True) + act_theme.toggled.connect(self._on_theme_toggle) + act_theme.setToolTip("Switch between the dark and light color theme.") + self._theme_act = act_theme + tools.addAction(act_theme) + + # ─── system tray / window lifecycle ──────────────────────────────── + def _tray_icon(self, state: str = "idle") -> QtGui.QIcon: + """Round accent-coloured badge; colour follows the agent state.""" + pm = QtGui.QPixmap(64, 64) + pm.fill(QtCore.Qt.GlobalColor.transparent) + p = QtGui.QPainter(pm) + p.setRenderHint(QtGui.QPainter.RenderHint.Antialiasing) + color = _STATE_COLORS.get(state, _CL_ACCENT) + if state == "idle": + color = _CL_ACCENT + p.setBrush(QtGui.QColor(color)) + p.setPen(QtCore.Qt.PenStyle.NoPen) + p.drawEllipse(4, 4, 56, 56) + p.setPen(QtGui.QColor("#ffffff")) + font = QtGui.QFont("Segoe UI", 30) + font.setBold(True) + p.setFont(font) + p.drawText(pm.rect(), QtCore.Qt.AlignmentFlag.AlignCenter, "J") + p.end() + return QtGui.QIcon(pm) + + def _build_tray(self): + self.setWindowIcon(self._tray_icon()) + if not QtWidgets.QSystemTrayIcon.isSystemTrayAvailable(): + return + tray = QtWidgets.QSystemTrayIcon(self._tray_icon(), self) + tray.setToolTip("Javis — idle") + menu = QtWidgets.QMenu() + act_show = menu.addAction("Show Javis") + act_show.triggered.connect(self._show_from_tray) + act_ptt = menu.addAction("Push to talk") + act_ptt.triggered.connect(self._push_to_talk) + act_stop = menu.addAction("Stop speaking") + act_stop.triggered.connect(self._interrupt) + menu.addSeparator() + act_quit = menu.addAction("Quit Javis") + act_quit.triggered.connect(self._quit) + tray.setContextMenu(menu) + tray.activated.connect(self._on_tray_activated) + tray.show() + self._tray = tray + + def _on_tray_activated(self, reason): + if reason in (QtWidgets.QSystemTrayIcon.ActivationReason.Trigger, + QtWidgets.QSystemTrayIcon.ActivationReason.DoubleClick): + if self.isVisible() and not self.isMinimized(): + self._hide_to_tray() + else: + self._show_from_tray() + + def _show_from_tray(self): + self.showNormal() + self.raise_() + self.activateWindow() + + def _hide_to_tray(self): + if self._tray is None: + self.showMinimized() + return + self.hide() + if not self._tray_hint_shown: + self._tray_hint_shown = True + self._tray.showMessage( + "Javis keeps listening", + "Wake word and timers stay active. Right-click the tray icon to quit.", + QtWidgets.QSystemTrayIcon.MessageIcon.Information, 4000) + + def _quit(self): + self._quit_requested = True + self.close() + + def closeEvent(self, event): # noqa: N802 + # Default close = hide to tray so the assistant keeps running. + if (not self._quit_requested and self._tray is not None + and config.gui.minimize_to_tray): + event.ignore() + self._hide_to_tray() + return + timers = [] + if self.orch is not None: + try: + timers = self.orch.timer_service.list_timers() + except Exception: # noqa: BLE001 + timers = [] + if timers: + names = ", ".join(t["label"] for t in timers[:4]) + more = f" (+{len(timers) - 4})" if len(timers) > 4 else "" + ans = QtWidgets.QMessageBox.question( + self, "Quit Javis?", + f"{len(timers)} timer(s) are still running: {names}{more}.\n\n" + "They will resume on the next start but won't ring while Javis is closed.\n" + "Quit anyway?", + QtWidgets.QMessageBox.StandardButton.Yes | QtWidgets.QMessageBox.StandardButton.No, + QtWidgets.QMessageBox.StandardButton.No) + if ans != QtWidgets.QMessageBox.StandardButton.Yes: + self._quit_requested = False + event.ignore() + return + if self._tray is not None: + self._tray.hide() + event.accept() + QtWidgets.QApplication.instance().quit() + + def _on_theme_toggle(self, checked: bool): + """checked=True → light, checked=False → dark.""" + _apply_theme("light" if checked else "dark") + + def _manage_models(self): + new_cfg = show_startup_dialog(config) + if new_cfg is None: + return + QtWidgets.QMessageBox.information( + self, "Restart required", + "Model and device changes take effect after restarting Javis.", + ) + + # ------------------------- handlers ------------------------- + + def _on_state(self, s: str): + self.state_badge.setText(f" {s} ") + self.state_badge.setStyleSheet(self._badge_style(s)) + self.state_dot.setStyleSheet(self._dot_style(s)) + self._set_pulse(s in ("listening", "speaking")) + if self._tray is not None: + self._tray.setToolTip(f"Javis — {s}") + self._tray.setIcon(self._tray_icon(s)) + self.lbl_state_big.setText(s) + self.lbl_state_big.setStyleSheet( + f"font-size: 20pt; font-weight: bold; padding: 4px; color: {_STATE_COLORS.get(s, '#888')};") + self.sb_phase.setText(f"state: {s}") + # PTT button reflects active recording state. + if s == "listening": + self._ptt_btn.setText("🔴 Recording… (Ctrl+Space)") + elif s in ("thinking", "speaking"): + self._ptt_btn.setText("⏳ Busy… (Ctrl+Space)") + else: + self._ptt_btn.setText("🎤 Push to Talk (Ctrl+Space)") + # Interrupt only makes sense while the agent is speaking. + self._interrupt_btn.setEnabled(s == "speaking") + + def _on_tool_call(self, name: str, args: str): + if self._show_tools.isChecked(): + self._append("tool→", f"{name}({args})") + self.sb_phase.setText(f"calling tool {name}…") + + def _on_tool_result(self, p: dict): + if self._show_tools.isChecked(): + status = "ok" if p.get("ok") else "ERR" + self._append("tool←", f"{p.get('name')} [{status}] {p.get('latency_s', 0):.2f}s — {p.get('preview', '')}") + + def _on_stt_done(self, p: dict): + if p.get("empty"): + self.lbl_stt_last.setText("") + self.lbl_stt_timing.setText("—") + self._append("error", "🎤 Keine Sprache erkannt — bitte wiederholen.") + return + text = p.get("text", "") + self.lbl_stt_last.setText(text[:300]) + # Store the last spoken text so Ctrl+R can re-send it. + if text.strip(): + self._last_sent_text = text.strip() + a = p.get("audio_s", 0.0); l = p.get("latency_s", 0.0) + rtf = (l / a) if a else 0.0 + self.lbl_stt_timing.setText(f"audio {a:.2f}s · stt {l:.2f}s · rtf {rtf:.2f}") + + def _on_llm_status(self, p: dict): + phase = p.get("phase", "?") + it = p.get("iter") + mx = p.get("max_iter") + if phase == "calling": + txt = f"calling (iter {it}/{mx}, tools={p.get('tools', 0)})" + elif phase == "response": + txt = (f"response (iter {it}/{mx}, tool_calls={p.get('tool_calls', 0)}, " + f"chars={p.get('content_len', 0)})") + self.lbl_llm_timing.setText(f"{p.get('latency_s', 0):.2f}s") + elif phase == "done": + txt = f"done (iter {it})" + else: + txt = phase + self.lbl_llm_phase.setText(txt) + self.sb_phase.setText(f"LLM: {txt}") + + def _on_tts_status(self, p: dict): + phase = p.get("phase", "?") + if phase == "begin": + total = p.get("sentences", 0) + self.tts_progress.setRange(0, max(1, total)) + self.tts_progress.setValue(0) + self.lbl_tts_phase.setText(f"speaking ({total} sentences)") + elif phase == "play": + self.tts_progress.setValue(p.get("idx", 0)) + self.tts_progress.setFormat(f"{p.get('idx', 0)}/{p.get('total', 0)}") + self.lbl_tts_timing.setText( + f"synth {p.get('synth_s', 0):.2f}s · audio {p.get('audio_s', 0):.2f}s") + self.lbl_tts_phase.setText( + f"playing: {p.get('text_preview', '')[:60]}") + elif phase == "done": + self.tts_progress.setValue(p.get("total", 0)) + self.lbl_tts_phase.setText("done") + elif phase == "skipped": + self.lbl_tts_phase.setText(f"skipped ({p.get('reason', '?')})") + elif phase == "cancelled": + self.lbl_tts_phase.setText("cancelled (interrupted)") + elif phase == "error": + self.lbl_tts_phase.setText(f"error: {p.get('error', '?')}") + + def _on_mcp_ready(self, snap: dict): + servers = snap.get("servers", []) + total = snap.get("tool_count", 0) + self.lbl_mcp_summary.setText( + f"{len(servers)} server(s) · {total} tool(s)") + self.mcp_tree.clear() + for s in servers: + root = QtWidgets.QTreeWidgetItem( + [s.get("name", "?"), f"{s.get('command', '')} (approval={s.get('approval', '?')})"]) + for tool in s.get("tools", []): + QtWidgets.QTreeWidgetItem(root, [tool, ""]) + self.mcp_tree.addTopLevelItem(root) + root.setExpanded(True) + + def _on_llm_health(self, p: dict): + ok = p.get("ok") + txt = "reachable ✓" if ok else "unreachable ✗" + self.lbl_llm_health.setText( + f"{txt}") + if not ok: + self._append( + "error", + f"⚠️ Ollama nicht erreichbar ({p.get('url', '')}) — " + "Anfragen werden fehlschlagen bis Ollama gestartet ist.", + ) + + def _on_mic_level(self, p: dict): + # rms in [0, ~1] → dB scale (−60..0 dB → 0..100). Much more + # readable than the previous linear ×400 boost. + import math + rms = float(p.get("rms", 0.0)) + score = float(p.get("score", 0.0)) + if rms <= 1e-5: + v = 0 + else: + db = 20.0 * math.log10(rms) + v = max(0, min(100, int((db + 60.0) * (100.0 / 60.0)))) + if abs(v - self._last_mic_val) >= 2: + self.mic_bar.setValue(v) + self.sb_mic.setValue(v) + self._set_bar_color(self.mic_bar, v) + self._set_bar_color(self.sb_mic, v) + self._last_mic_val = v + sv = max(0, min(100, int(score * 100))) + if abs(sv - self._last_wake_val) >= 2: + self.wake_bar.setValue(sv) + self.sb_wake.setValue(sv) + self._set_bar_color(self.wake_bar, sv) + self._set_bar_color(self.sb_wake, sv) + self._last_wake_val = sv + + @staticmethod + def _set_bar_color(bar: "QtWidgets.QProgressBar", v: int): + """Paint a progress-bar chunk green→yellow→red as it fills.""" + bar.setStyleSheet( + f"QProgressBar::chunk{{background:{_level_color(v)};}}") + + def _on_wake(self, score: float): + self.lbl_last_wake.setText( + f"{time.strftime('%H:%M:%S')} (score {score:.2f})") + + def _on_ingest(self, p: dict): + phase = p.get("phase", "?") + tid = f"ingest:{p.get('id', '?')}" + if phase in ("loading", "loaded", "segmenting", "segmented", "loading_stt"): + self._task_upsert(tid, f"Audiobook ingest ({p.get('voice', '?')})", phase) + self.voice_status.setText(f"{p.get('voice')}: {phase} …") + elif phase == "resuming": + from_idx = p.get("from_index", 0); total = p.get("total", 1) + pct = int(100 * from_idx / max(1, total)) + self._task_upsert(tid, f"Audiobook ingest ({p.get('voice', '?')})", + f"resuming {from_idx}/{total}") + self.voice_status.setText( + f"{p.get('voice')}: resuming from {from_idx}/{total} ({pct}%)") + self.ingest_progress.show() + self.ingest_progress.setRange(0, total) + self.ingest_progress.setValue(from_idx) + elif phase == "processing": + done = p.get("done", 0); total = p.get("total", 1) + pct = int(100 * done / max(1, total)) + self._task_upsert(tid, f"Audiobook ingest ({p.get('voice', '?')})", + f"{done}/{total} refs={p.get('refs', 0)} train={p.get('train', 0)}") + self.voice_status.setText( + f"{p.get('voice')}: {done}/{total} segments " + f"({p.get('refs', 0)} refs · {p.get('train', 0)} train · {p.get('rejected', 0)} skip)") + self.ingest_progress.show() + self.ingest_progress.setRange(0, total) + self.ingest_progress.setValue(done) + self.ingest_progress.setFormat(f"{pct}% ({done}/{total})") + elif phase == "stopped": + at = p.get("at_index", 0); total = p.get("total", 1) + pct = int(100 * at / max(1, total)) + self._task_finish(tid, ok=True, + msg=f"stopped at {at}/{total} — checkpoint saved") + self.voice_status.setText( + f"stopped at {at}/{total} ({pct}%) — checkpoint saved, resume next time") + self.ingest_progress.hide() + self.ingest_stop_btn.setEnabled(True) + self.ingest_stop_btn.hide() + elif phase == "done": + self._task_finish(tid, ok=True, + msg=f"refs={p.get('refs')} train={p.get('train')}") + self.voice_status.setText( + f"done: {p.get('train', 0)} clips · {p.get('refs', 0)} refs") + self.ingest_progress.setValue(self.ingest_progress.maximum()) + # Auto-hide after a short delay so the user sees the final state. + # Skip the timer when more files remain in a batch. + if getattr(self, '_ingest_queue_size', 0) == 0: + QtCore.QTimer.singleShot(1500, self.ingest_progress.hide) + self._refresh_voices() + + def _on_error(self, p: dict): + # Close any open streaming bubble so the error renders cleanly. + if self._assistant_in_progress: + self._finalize_assistant_bubble() + where = p.get("where", "?"); msg = p.get("message", "") + ts = time.strftime("%H:%M:%S") + self.lbl_last_error.setText(f"[{ts}] {where}: {msg}") + # Also surface the error inline in the conversation view so the user + # doesn't have to switch to the Status tab to notice it. + self._append("error", f"⚠️ {where}: {msg}") + if self._tray is not None and not self.isVisible(): + self._tray.showMessage("Javis — error", f"{where}: {msg}", + QtWidgets.QSystemTrayIcon.MessageIcon.Warning, 6000) + + def _on_notice(self, p: dict): + """Non-error status hints (busy, queued, config) — status bar only.""" + msg = str(p.get("message", "")) + if msg: + self.statusBar().showMessage(msg, 5000) + + def _on_task_start(self, p: dict): + self._task_upsert(p.get("id", "?"), p.get("name", "?"), "running") + + def _on_task_done(self, p: dict): + self._task_finish(p.get("id", "?"), ok=bool(p.get("ok", True)), + msg=str(p.get("msg", ""))) + + # ------------------------- task table ------------------------- + + def _task_upsert(self, tid: str, name: str, status: str): + if tid not in self._tasks: + self._tasks[tid] = {"name": name, "started": time.monotonic(), + "status": status, "done": False} + row = self.task_table.rowCount() + self.task_table.insertRow(row) + self.task_table.setItem(row, 0, QtWidgets.QTableWidgetItem(name)) + self.task_table.setItem(row, 1, QtWidgets.QTableWidgetItem(status)) + self.task_table.setItem(row, 2, QtWidgets.QTableWidgetItem("0.0s")) + self._tasks[tid]["row"] = row + else: + self._tasks[tid]["status"] = status + row = self._tasks[tid]["row"] + self.task_table.item(row, 1).setText(status) + self._refresh_task_count() + + def _task_finish(self, tid: str, ok: bool, msg: str): + if tid not in self._tasks: + # finished a task we never started — synthesize a row + self._task_upsert(tid, tid, "done" if ok else "failed") + t = self._tasks[tid] + t["done"] = True + row = t["row"] + status = ("✓ " if ok else "✗ ") + (msg or ("done" if ok else "failed")) + self.task_table.item(row, 1).setText(status) + elapsed = time.monotonic() - t["started"] + self.task_table.item(row, 2).setText(self._fmt_elapsed(elapsed)) + self._prune_finished_tasks() + self._refresh_task_count() + + @staticmethod + def _fmt_elapsed(s: float) -> str: + if s < 60: + return f"{s:.1f}s" + m, s = divmod(int(s), 60) + return f"{m}m{s:02d}s" + + def _prune_finished_tasks(self): + done_items = [(tid, t) for tid, t in self._tasks.items() if t["done"]] + if len(done_items) <= _TASKS_MAX_ROWS: + return + # Sort by row index; remove oldest finished rows first. + done_items.sort(key=lambda kv: kv[1]["row"]) + to_remove = done_items[: len(done_items) - _TASKS_MAX_ROWS] + removed_rows = sorted({t["row"] for _, t in to_remove}, reverse=True) + for r in removed_rows: + self.task_table.removeRow(r) + removed_set = {tid for tid, _ in to_remove} + for tid in removed_set: + self._tasks.pop(tid, None) + # Re-index remaining task rows (row numbers shift after removal). + for tid, t in self._tasks.items(): + old = t["row"] + shift = sum(1 for r in removed_rows if r < old) + t["row"] = old - shift + + def _refresh_task_count(self): + active = sum(1 for t in self._tasks.values() if not t["done"]) + self.sb_tasks.setText(f"tasks: {active}") + + def _refresh_task_elapsed(self): + now = time.monotonic() + for t in self._tasks.values(): + if t["done"]: + continue + row = t["row"] + self.task_table.item(row, 2).setText(self._fmt_elapsed(now - t['started'])) + + def _refresh_timers(self): + """Rebuild timer chips from live timer service state (called every 500 ms).""" + if self.orch is None: + return + timer_service = getattr(self.orch, "timer_service", None) + if timer_service is None: + self._timer_bar.hide() + return + timers = timer_service.list_timers() + # Clear existing chips. + while self._timer_chips_layout.count(): + item = self._timer_chips_layout.takeAt(0) + if item and item.widget(): + item.widget().deleteLater() + self._timer_chip_map.clear() + if not timers: + self._timer_bar.hide() + return + self._timer_bar.show() + for t in timers: + remaining = t["seconds_remaining"] + mins, secs = divmod(remaining, 60) + hrs, mins = divmod(mins, 60) + if hrs: + time_str = f"{hrs}:{mins:02d}:{secs:02d}" + else: + time_str = f"{mins}:{secs:02d}" + label_text = t["label"] or _format_timer_label(t["duration_s"]) + chip = QtWidgets.QFrame() + chip.setStyleSheet( + f"QFrame {{ background: {_CL_SURFACE}; border: 1px solid {_CL_SUCCESS_BORDER}; " + f"border-radius: 10px; }}" + ) + chip_lay = QtWidgets.QHBoxLayout(chip) + chip_lay.setContentsMargins(8, 2, 6, 2) + chip_lay.setSpacing(4) + chip_lbl = QtWidgets.QLabel(f"{label_text} {time_str}") + chip_lbl.setStyleSheet( + f"color: {_CL_TEXT}; font-size: 9.5pt; background: transparent; border: 0;" + ) + chip_lay.addWidget(chip_lbl) + cancel_btn = QtWidgets.QPushButton("×") + cancel_btn.setFixedSize(16, 16) + cancel_btn.setToolTip(f"Cancel {label_text} timer") + cancel_btn.setStyleSheet( + f"QPushButton {{ background: transparent; color: {_CL_MUTED}; border: 0; " + f" font-size: 11pt; font-weight: bold; padding: 0; }}" + f"QPushButton:hover {{ color: {_CL_ERROR}; }}" + ) + _tid = t["id"] + cancel_btn.clicked.connect(lambda _, tid=_tid: self._cancel_timer(tid)) + chip_lay.addWidget(cancel_btn) + self._timer_chips_layout.addWidget(chip) + self._timer_chip_map[t["id"]] = chip + + def _cancel_timer(self, timer_id: str): + if self.orch is None: + return + self.orch.timer_service.cancel_timer(timer_id) + self._refresh_timers() + + def _on_timer_fired(self, payload: dict): + label = payload.get("label", "") + self.sb_phase.setText(f"⏰ {label} Timer fertig!") + # Flash the status bar message for 5 s then revert. + QtCore.QTimer.singleShot( + 5000, lambda: self.sb_phase.setText(f"state: {self.state_badge.text().strip()}") + ) + self._refresh_timers() + + # ------------------------- voices tab actions ------------------------- + + def _populate_voice_combo(self): + """Refresh the voice dropdown from disk; preserve current selection.""" + prev = self._voice_combo.currentText() + self._voice_combo.blockSignals(True) + self._voice_combo.clear() + names = [p.name for p in VoiceManager(config).list()] + for name in names: + self._voice_combo.addItem(name) + # Prefer: config value → previous selection → first item. + target = config.tts.default_voice + if target not in names: + target = prev if prev in names else (names[0] if names else "") + idx = self._voice_combo.findText(target) + if idx >= 0: + self._voice_combo.setCurrentIndex(idx) + self._voice_combo.blockSignals(False) + # Keep test combo in sync (guarded — built after conv combo) + if hasattr(self, "_tts_test_combo"): + prev_t = self._tts_test_combo.currentText() + self._tts_test_combo.blockSignals(True) + self._tts_test_combo.clear() + for name in names: + self._tts_test_combo.addItem(name) + # Preserve whatever profile was selected in the test combo. + idx_t = self._tts_test_combo.findText(prev_t) + if idx_t >= 0: + self._tts_test_combo.setCurrentIndex(idx_t) + self._tts_test_combo.blockSignals(False) + + def _on_voice_changed(self, name: str): + if name: + config.tts.default_voice = name + + def _run_tts_test(self): + """Synthesize the test text with the selected voice and play it.""" + text = self._tts_test_input.text().strip() + if not text: + self._tts_test_status.setText("⚠ Please enter some text first.") + return + if not self.orch or not self.orch.tts: + self._tts_test_status.setText("⚠ TTS not loaded — wait for startup to finish.") + return + voice_name = self._tts_test_combo.currentText() + if not voice_name: + self._tts_test_status.setText("⚠ No voice profile selected.") + return + + self._tts_test_btn.setEnabled(False) + self._tts_test_status.setText("Synthesizing…") + self._tts_test_status.setStyleSheet(f"color:{_CL_WARN_FG}; font-size:10pt;") + + async def _do(): + try: + from ..audio_io.stream import play_pcm + profile = VoiceManager(config).get(voice_name) + result = await self.orch.tts.synthesize(text, profile) + await play_pcm( + result.audio, result.sample_rate, + config.audio.output_device, + ) + self._tts_test_status.setText("✓ Done") + self._tts_test_status.setStyleSheet(f"color:{_CL_SUCCESS}; font-size:10pt;") + except Exception as exc: + self._tts_test_status.setText(f"⚠ {exc}") + self._tts_test_status.setStyleSheet(f"color:{_CL_ERROR}; font-size:10pt;") + log.exception("TTS test failed") + finally: + self._tts_test_btn.setEnabled(True) + + asyncio.ensure_future(_do()) + + def _refresh_voices(self): + mgr = VoiceManager(config) + profiles = mgr.list() + self.voice_list.clear() + for p in profiles: + item = QtWidgets.QListWidgetItem( + f"{p.name} ({p.language}, {len(p.reference_clips)} refs)" + ) + # Store the actual profile name so display formatting can change. + item.setData(QtCore.Qt.UserRole, p.name) + self.voice_list.addItem(item) + self._populate_voice_combo() + # Onboarding guidance for new users. + if not profiles: + self.voice_hint.setText( + "No voice profiles yet. " + "Click \u201eNew profile\u201c to create one, " + "then use \u201eIngest audiobook\u2026\u201c to add training data." + ) + self.voice_hint.show() + elif any(not (config.paths.voices_dir / p.name / "piper_model.onnx").exists() for p in profiles): + untrained = [ + p.name for p in profiles + if not (config.paths.voices_dir / p.name / "piper_model.onnx").exists() + ] + self.voice_hint.setText( + f"Profile(s) without a trained model: {', '.join(untrained)}. " + "Ingest audiobooks to create training data, then click \u201eTrain model\u2026\u201c." + ) + self.voice_hint.show() + else: + self.voice_hint.hide() + # Refresh piper status for currently selected profile. + sel_item = self.voice_list.currentItem() + sel_name = sel_item.data(QtCore.Qt.UserRole) if sel_item else None + self._refresh_piper_status(sel_name) + + def _new_voice(self): + name, ok = QtWidgets.QInputDialog.getText(self, "New voice", "Profile name:") + if not (ok and name): return + name = name.strip() + if not name: return + # Ask for an optional German base checkpoint. + lang_items = ["de (German)", "en (English)", "other"] + lang_choice, ok2 = QtWidgets.QInputDialog.getItem( + self, "Language", + "Primary language for this voice:", + lang_items, 0, False, + ) + language = "de" if lang_choice.startswith("de") else ("en" if lang_choice.startswith("en") else "") + base_ckpt = "" + if language == "de": + base_ckpt, _ = QtWidgets.QInputDialog.getText( + self, "Optional: German base checkpoint", + "Path to a pre-trained German Piper checkpoint for fine-tuning\n" + "(leave empty to train from scratch):", + ) + base_ckpt = base_ckpt.strip() + try: + mgr = VoiceManager(config) + mgr.create(name) + # Persist language into profile.json + prof = mgr.get(name) + prof.language = language or "de" + prof.save() + if base_ckpt: + # Store the checkpoint path in a training config file for later use. + import json as _json + ckpt_cfg = config.paths.voices_dir / name / "training" / "base_checkpoint.txt" + ckpt_cfg.parent.mkdir(parents=True, exist_ok=True) + ckpt_cfg.write_text(base_ckpt, encoding="utf-8") + except Exception as e: + log.exception(f"new voice profile '{name}' failed") + QtWidgets.QMessageBox.warning(self, "Error", str(e)) + self._refresh_voices() + + def _selected_voice_name(self) -> str | None: + item = self.voice_list.currentItem() + if not item: + return None + return item.data(QtCore.Qt.UserRole) or item.text().split()[0] + + def _export_voice(self): + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile to export.") + return + default_name = f"{name}.javisvoice" + path, _ = QtWidgets.QFileDialog.getSaveFileName( + self, "Export voice profile", default_name, + "Javis voice (*.javisvoice);;Zip archive (*.zip)", + ) + if not path: + return + self.voice_status.setText(f"exporting '{name}' …") + self.ingest_progress.show() + self.ingest_progress.setRange(0, 0) # busy spinner + + async def _do(): + try: + result = await asyncio.to_thread( + VoiceManager(config).export, name, Path(path), + ) + except Exception as e: + log.exception("export failed") + self.voice_status.setText(f"export failed: {e}") + QtWidgets.QMessageBox.warning(self, "Export failed", str(e)) + else: + mb = result.bytes / (1024 * 1024) + self.voice_status.setText( + f"exported '{result.profile_name}' " + f"({result.files} files · {mb:.1f} MB) → {result.path}" + ) + finally: + self.ingest_progress.setRange(0, 1) + QtCore.QTimer.singleShot(1500, self.ingest_progress.hide) + + asyncio.ensure_future(_do()) + + def _import_voice(self): + path, _ = QtWidgets.QFileDialog.getOpenFileName( + self, "Import voice profile", "", + "Javis voice (*.javisvoice *.zip);;All files (*.*)", + ) + if not path: + return + archive = Path(path) + try: + manifest = VoiceManager.inspect_archive(archive) + except Exception as e: + QtWidgets.QMessageBox.warning(self, "Invalid archive", str(e)) + return + + suggested = manifest.get("name") or archive.stem + new_name, ok = QtWidgets.QInputDialog.getText( + self, "Import voice profile", + f"Import as profile name:\n(from {archive.name}, " + f"language={manifest.get('language', '?')})", + text=str(suggested), + ) + if not ok: + return + new_name = new_name.strip() + if not new_name: + return + + overwrite = False + if (config.paths.voices_dir / new_name).exists(): + resp = QtWidgets.QMessageBox.question( + self, "Profile exists", + f"A profile named '{new_name}' already exists.\n\n" + f"Yes: overwrite it.\nNo: import under a new name " + f"(e.g. '{new_name}_2').", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No + | QtWidgets.QMessageBox.Cancel, + ) + if resp == QtWidgets.QMessageBox.Cancel: + return + overwrite = (resp == QtWidgets.QMessageBox.Yes) + + self.voice_status.setText(f"importing {archive.name} …") + self.ingest_progress.show() + self.ingest_progress.setRange(0, 0) + + async def _do(): + try: + result = await asyncio.to_thread( + VoiceManager(config).import_archive, + archive, new_name, overwrite, + ) + except Exception as e: + log.exception("import failed") + self.voice_status.setText(f"import failed: {e}") + QtWidgets.QMessageBox.warning(self, "Import failed", str(e)) + else: + mb = result.bytes / (1024 * 1024) + rename_note = (f" (renamed from '{result.renamed_from}')" + if result.renamed_from else "") + self.voice_status.setText( + f"imported '{result.profile_name}'{rename_note} " + f"— {result.files} files · {mb:.1f} MB" + ) + self._refresh_voices() + finally: + self.ingest_progress.setRange(0, 1) + QtCore.QTimer.singleShot(1500, self.ingest_progress.hide) + + asyncio.ensure_future(_do()) + + def _delete_voice(self): + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile to delete.") + return + resp = QtWidgets.QMessageBox.question( + self, "Delete voice profile", + f"Permanently delete profile '{name}'?\n\n" + f"This removes reference clips and training data.", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, + ) + if resp != QtWidgets.QMessageBox.Yes: + return + try: + VoiceManager(config).delete(name) + except Exception as e: + log.exception(f"delete voice '{name}' failed") + QtWidgets.QMessageBox.warning(self, "Delete failed", str(e)) + self._refresh_voices() + + # ── Piper training ───────────────────────────────────────────────────── + + def _refresh_piper_status(self, voice_name: str | None) -> None: + """Update the Piper model status label for the selected profile.""" + if not voice_name: + self._piper_status_lbl.setText("Select a profile above.") + return + voice_dir = config.paths.voices_dir / voice_name + onnx = voice_dir / "piper_model.onnx" + meta = voice_dir / "training" / "metadata.csv" + n_clips = sum(1 for _ in meta.open(encoding="utf-8", errors="ignore")) if meta.exists() else 0 + ckpt_file = voice_dir / "training" / "base_checkpoint.txt" + base_ckpt = ckpt_file.read_text(encoding="utf-8").strip() if ckpt_file.exists() else "" + if onnx.exists(): + size_mb = onnx.stat().st_size / (1024 * 1024) + status = f"✓ Trained model ready ({size_mb:.1f} MB) · {n_clips} training utterances" + else: + status = f"Not trained yet · {n_clips} training utterances available" + if base_ckpt: + status += f"\nBase checkpoint: {base_ckpt}" + self._piper_status_lbl.setText(status) + + def _train_piper_model(self) -> None: + """Launch Piper VITS training for the selected voice profile.""" + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile first.") + return + voice_dir = config.paths.voices_dir / name + + # Block a second concurrent run (e.g. the overnight scheduler is + # already training the same voice). + try: + from ..voices.training_scheduler import training_in_progress + if training_in_progress(voice_dir): + QtWidgets.QMessageBox.warning( + self, "Training already running", + f"Training for '{name}' is already in progress.\n\n" + "Wait for it to finish before starting another run." + ) + return + except Exception: + pass + + meta = voice_dir / "training" / "metadata.csv" + if not meta.exists(): + QtWidgets.QMessageBox.warning( + self, "No training data", + f"No training data found for '{name}'.\n\n" + "Use 'Ingest audiobook\u2026' to create training clips first." + ) + return + n_clips = sum(1 for _ in meta.open(encoding="utf-8", errors="ignore")) + # Read saved base checkpoint if any. + ckpt_file = voice_dir / "training" / "base_checkpoint.txt" + default_ckpt = ckpt_file.read_text(encoding="utf-8").strip() if ckpt_file.exists() else "" + base_ckpt, ok = QtWidgets.QInputDialog.getText( + self, "Train Piper model", + f"Profile: {name} ({n_clips} utterances)\n\n" + "Optional: path to a pre-trained German Piper checkpoint\n" + "(leave empty to train from scratch):", + text=default_ckpt, + ) + if not ok: + return + base_ckpt = base_ckpt.strip() + # Persist any new checkpoint choice. + if base_ckpt: + ckpt_file.parent.mkdir(parents=True, exist_ok=True) + ckpt_file.write_text(base_ckpt, encoding="utf-8") + + # Batch size default was previously hardcoded (8), leaving most of a + # 24 GB GPU idle — now user-tunable per run, saved for next time. + bsz_file = voice_dir / "training" / "batch_size.txt" + default_bsz = 32 + if bsz_file.exists(): + try: + default_bsz = int(bsz_file.read_text(encoding="utf-8").strip()) + except ValueError: + pass + batch_size, ok = QtWidgets.QInputDialog.getInt( + self, "Train Piper model", + "Batch size (default 32). Batches group clips of similar length,\n" + "so VRAM stays roughly flat across the run. On a 24 GB card with\n" + "clips up to 15 s, 32 clears every bucket, while 40 runs out of\n" + "memory on the longest ones.\n" + "Lower this if you still hit an out-of-memory error:", + default_bsz, 1, 512, 1, + ) + if not ok: + return + bsz_file.parent.mkdir(parents=True, exist_ok=True) + bsz_file.write_text(str(batch_size), encoding="utf-8") + + self.voice_status.setText(f"Launching Piper training for '{name}'…") + import sys as _sys + import subprocess as _subprocess + cmd = [ + _sys.executable, "-m", "javis.voices.piper_train", + "--voice", name, + "--batch-size", str(batch_size), + ] + if base_ckpt: + cmd += ["--base-checkpoint", base_ckpt] + try: + from ..core.config import find_project_root + _project_root = str(find_project_root()) + # Wrap in `cmd /k` so the console stays open on errors/completion. + _subprocess.Popen( + ["cmd", "/k", _subprocess.list2cmdline(cmd)], + creationflags=getattr(_subprocess, "CREATE_NEW_CONSOLE", 0), + cwd=_project_root, + ) + self.voice_status.setText( + f"Training '{name}' launched in a new terminal. " + "This takes several hours. The model appears in the voice profile when done." + ) + except Exception as exc: + log.exception("piper_train launch failed") + QtWidgets.QMessageBox.critical(self, "Launch failed", str(exc)) + + def _export_piper_snapshot(self) -> None: + """Export the best existing training checkpoint to ONNX right now.""" + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile first.") + return + voice_dir = config.paths.voices_dir / name + lightning_dir = voice_dir / "training" / "piper_lightning" + if not lightning_dir.exists(): + QtWidgets.QMessageBox.warning( + self, "No training checkpoints", + f"No training data found for '{name}'.\n\n" + "Start training first with the 'Train model\u2026' button." + ) + return + + import sys as _sys, pathlib as _pathlib, subprocess as _subprocess + from ..voices.piper_train import _find_best_checkpoint + ckpt = _find_best_checkpoint(lightning_dir) + if ckpt is None: + QtWidgets.QMessageBox.warning( + self, "No checkpoint found", + f"No .ckpt files found under:\n{lightning_dir}\n\n" + "Start training first." + ) + return + + onnx_out = voice_dir / "piper_model.onnx" + msg = ( + f"Export checkpoint to ONNX?\n\n" + f"Source: {ckpt.name}\n" + f"Output: {onnx_out}\n\n" + f"This will overwrite any existing model file.\n" + f"Training can continue independently afterwards." + ) + if QtWidgets.QMessageBox.question( + self, "Export snapshot", msg, + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, + ) != QtWidgets.QMessageBox.Yes: + return + + self.voice_status.setText(f"Exporting snapshot for '{name}'…") + self.voice_status.repaint() + + cmd = [ + _sys.executable, "-m", "javis.voices.piper_train", + "--voice", name, + "--export-snapshot", + ] + try: + from ..core.config import find_project_root + _project_root = str(find_project_root()) + _subprocess.Popen( + ["cmd", "/k", _subprocess.list2cmdline(cmd)], + creationflags=getattr(_subprocess, "CREATE_NEW_CONSOLE", 0), + cwd=_project_root, + ) + self.voice_status.setText( + f"Export for '{name}' launched in a new terminal." + ) + except Exception as exc: + log.exception("export piper snapshot subprocess failed") + QtWidgets.QMessageBox.critical(self, "Export failed", str(exc)) + + def _activate_piper_voice(self) -> None: + """Set the selected profile as the active Piper TTS voice.""" + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile first.") + return + voice_dir = config.paths.voices_dir / name + onnx = voice_dir / "piper_model.onnx" + if not onnx.exists(): + ans = QtWidgets.QMessageBox.question( + self, "No model yet", + f"'{name}' has no exported model yet (piper_model.onnx).\n\n" + "Export a snapshot first?", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, + ) + if ans == QtWidgets.QMessageBox.Yes: + self._export_piper_snapshot() + return + + # Update the in-memory config. + config.tts.default_voice = name + try: + cfg_path = config.source_path + if cfg_path is None: + raise RuntimeError("No config.yaml loaded — restart with a config file.") + update_config_file(cfg_path, {"tts": {"default_voice": name}}) + except Exception as exc: + log.exception("voice activation config save failed") + QtWidgets.QMessageBox.warning( + self, "Config save failed", + f"Voice set in memory but could not persist to config.yaml:\n{exc}" + ) + + # Reload TTS backend live if the orchestrator is running. + self.voice_status.setText( + f"\u2713 Active voice set to '{name}' \u2014 restart Javis to apply." + ) + self._refresh_piper_status(name) + + # ── reference clip management ────────────────────────────────────────── + + def _refresh_refs(self, voice_name: str | None) -> None: + """Populate refs_list with the first _REFS_PAGE clips; more on demand.""" + self.refs_list.clear() + self._refs_all_wavs: list = [] + self._refs_loaded_count = 0 + self._refs_load_more_btn.hide() + if not voice_name: + self.refs_hint_lbl.setText("Select a profile above to manage its reference clips.") + return + refs_dir = config.paths.voices_dir / voice_name / "refs" + all_wavs = sorted(refs_dir.glob("*.wav")) if refs_dir.exists() else [] + if not all_wavs: + self.refs_hint_lbl.setText( + f"No clips yet. Click \u201cAdd clip\u2026\u201d to add WAV files to '{voice_name}'." + ) + return + total = len(all_wavs) + self.refs_hint_lbl.setText( + f"{total} clip(s) — aim for 5\u201330 s each, 5\u201315 total" + ) + self._refs_all_wavs = all_wavs + self._refs_load_more() + + def _refs_load_more(self) -> None: + """Append the next page of reference clips to refs_list.""" + wavs = getattr(self, "_refs_all_wavs", []) + start = getattr(self, "_refs_loaded_count", 0) + page = wavs[start : start + _REFS_PAGE] + try: + import soundfile as sf + for wav in page: + try: + info = sf.info(str(wav)) + label = f"{wav.name} ({info.duration:.1f} s, {info.samplerate} Hz)" + except Exception: + label = wav.name + item = QtWidgets.QListWidgetItem(label) + item.setData(QtCore.Qt.UserRole, str(wav)) + self.refs_list.addItem(item) + except ImportError: + for wav in page: + item = QtWidgets.QListWidgetItem(wav.name) + item.setData(QtCore.Qt.UserRole, str(wav)) + self.refs_list.addItem(item) + self._refs_loaded_count = start + len(page) + remaining = len(wavs) - self._refs_loaded_count + if remaining > 0: + self._refs_load_more_btn.setText( + f"Load {min(_REFS_PAGE, remaining)} more\u2026 ({remaining} remaining)" + ) + self._refs_load_more_btn.show() + else: + self._refs_load_more_btn.hide() + + def _add_ref(self) -> None: + """Open a file dialog and copy selected WAV(s) into the profile's refs/ folder.""" + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile first.") + return + paths, _ = QtWidgets.QFileDialog.getOpenFileNames( + self, "Add reference clip(s)", "", + "WAV audio (*.wav);;All files (*.*)", + ) + if not paths: + return + refs_dir = config.paths.voices_dir / name / "refs" + refs_dir.mkdir(parents=True, exist_ok=True) + import shutil as _shutil + added = 0 + for path in paths: + src = Path(path) + dst = refs_dir / src.name + # Avoid silent overwrite — append _N suffix if file exists. + if dst.exists(): + n = 1 + while True: + candidate = refs_dir / f"{src.stem}_{n}{src.suffix}" + if not candidate.exists(): + dst = candidate + break + n += 1 + _shutil.copy2(str(src), str(dst)) + added += 1 + # Refresh the voice list so any newly-added clip is reflected. + self.voice_status.setText(f"Added {added} clip(s) to '{name}'.") + self._refresh_refs(name) + self._refresh_voices() + + def _delete_ref(self) -> None: + """Remove the selected reference clip from disk.""" + name = self._selected_voice_name() + item = self.refs_list.currentItem() + if not item: + QtWidgets.QMessageBox.information( + self, "Select a clip", "Select a reference clip first.") + return + wav_path = Path(item.data(QtCore.Qt.UserRole)) + resp = QtWidgets.QMessageBox.question( + self, "Delete clip", + f"Permanently remove '{wav_path.name}' from profile '{name}'?\n\n" + f"This cannot be undone.", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, + ) + if resp != QtWidgets.QMessageBox.Yes: + return + try: + wav_path.unlink() + except Exception as exc: + log.exception(f"delete ref clip '{wav_path.name}' failed") + QtWidgets.QMessageBox.warning(self, "Delete failed", str(exc)) + return + self.voice_status.setText(f"Deleted '{wav_path.name}'.") + self._refresh_refs(name) + self._refresh_voices() + + def _play_ref(self) -> None: + """Preview the selected reference clip through the audio output device.""" + item = self.refs_list.currentItem() + if not item: + return + wav_path = Path(item.data(QtCore.Qt.UserRole)) + + async def _do() -> None: + try: + import soundfile as sf + from ..audio_io.stream import play_pcm + audio, sr = sf.read(str(wav_path), dtype="float32", always_2d=False) + if audio.ndim > 1: + audio = audio.mean(axis=1) + self.voice_status.setText(f"Playing '{wav_path.name}'…") + await play_pcm(audio, sr, config.audio.output_device) + self.voice_status.setText("Done.") + except Exception as exc: + log.exception(f"failed to play reference clip '{wav_path.name}'") + self.voice_status.setText(f"Playback failed: {exc}") + + asyncio.ensure_future(_do()) + + def _ingest_audiobook(self): + name = self._selected_voice_name() + if not name: + QtWidgets.QMessageBox.information( + self, "Select a profile", "Select a voice profile first.") + return + paths, _ = QtWidgets.QFileDialog.getOpenFileNames( + self, "Choose audiobook files", + filter="Audio (*.mp3 *.wav *.flac *.m4a *.ogg)") + if not paths: + return + + # Sort by filename so chapters are processed in order. + sources = sorted([Path(p) for p in paths], key=lambda p: p.name) + n = len(sources) + + # Ask about checkpoints for each file upfront so the batch runs unattended. + resume_map: dict[Path, bool] = {} + for source in sources: + cp = checkpoint_info(config, name, source) + if cp is not None: + pct = int(100 * cp["next_index"] / max(1, cp["total"])) + msg = ( + f"A checkpoint was found for {source.name}.

" + f"Progress: {cp['next_index']}/{cp['total']} segments ({pct}%)
" + f"Saved: {cp['saved_at']}

" + f"Resume from the saved position, or start over?" + ) + dlg = QtWidgets.QMessageBox(self) + dlg.setWindowTitle("Resume ingest?") + dlg.setText(msg) + dlg.setTextFormat(QtCore.Qt.RichText) + btn_resume = dlg.addButton("Resume", QtWidgets.QMessageBox.AcceptRole) + dlg.addButton("Start over", QtWidgets.QMessageBox.DestructiveRole) + cancel_btn = dlg.addButton("Cancel", QtWidgets.QMessageBox.RejectRole) + dlg.setDefaultButton(btn_resume) + dlg.exec() + clicked = dlg.clickedButton() + if clicked is None or clicked is cancel_btn: + return + resume_map[source] = (clicked is btn_resume) + else: + resume_map[source] = False + + label = f"{sources[0].name} (1/{n})" if n > 1 else sources[0].name + self.voice_status.setText(f"ingesting {label} …") + self.ingest_progress.show() + self.ingest_progress.setRange(0, 0) # busy indicator until segments known + self.ingest_stop_btn.show() + + stop_ev = asyncio.Event() + self._ingest_stop_event = stop_ev + bus = self.orch.bus if self.orch is not None else None + + async def _do(): + try: + for i, source in enumerate(sources, 1): + if stop_ev.is_set(): + break + # Set files remaining BEFORE starting so _on_ingest sees + # the correct count when the "done" event fires. + self._ingest_queue_size = n - i + if n > 1: + self.voice_status.setText( + f"ingesting {source.name} ({i}/{n}) …") + self.ingest_progress.show() + self.ingest_progress.setRange(0, 0) + try: + await ingest_audiobook( + config, source, name, + max_reference_clips=9999, + bus=bus, task_id=f"{name}:{source.stem}", + stop_event=stop_ev, resume=resume_map[source], + ) + except Exception as e: + remaining = n - i + suffix = f" — skipping ({remaining} remaining)" if remaining else "" + self.voice_status.setText(f"failed: {source.name}: {e}{suffix}") + log.exception(f"ingest failed for {source.name}") + self.ingest_progress.hide() + finally: + self._ingest_stop_event = None + self._ingest_queue_size = 0 + self.ingest_stop_btn.hide() + asyncio.ensure_future(_do()) + + def _stop_ingest(self): + if self._ingest_stop_event is not None: + self._ingest_stop_event.set() + self.ingest_stop_btn.setEnabled(False) + self.voice_status.setText("stopping — finishing current segment …") + + # ------------------------- chat rendering ------------------------- + + @staticmethod + def _esc_html(s: str) -> str: + return (s.replace("&", "&").replace("<", "<") + .replace(">", ">").replace("\n", "
")) + + @staticmethod + def _md_to_html(s: str) -> str: + """Convert LLM markdown to HTML for display in chat bubbles. + + HTML-escapes raw chars first, then applies markdown transformations + in order from most-specific to least-specific so inner spans are + processed before outer ones (e.g. bold inside a heading). + """ + # 1. HTML-escape before inserting any tags + s = s.replace("&", "&").replace("<", "<").replace(">", ">") + # 2. Fenced code blocks → 
+            s = re.sub(
+                r'```[a-z]*\n?([\s\S]*?)```',
+                lambda m: (
+                    f'
{m.group(1).rstrip()}
'
+                ),
+                s,
+            )
+            # 3. Inline code → 
+            s = re.sub(
+                r'`([^`]+)`',
+                lambda m: (
+                    f'{m.group(1)}'
+                ),
+                s,
+            )
+            # 4. Bold + italic (***text***)
+            s = re.sub(r'\*\*\*([^*\n]+)\*\*\*', r'\1', s)
+            # 5. Bold (**text**)
+            s = re.sub(r'\*\*([^*\n]+)\*\*', r'\1', s)
+            # 6. Italic (*text*)
+            s = re.sub(r'\*([^*\n]+)\*', r'\1', s)
+            # 7. Bold (__text__) / Italic (_text_)
+            s = re.sub(r'__([^_\n]+)__', r'\1', s)
+            s = re.sub(r'_([^_\n]+)_', r'\1', s)
+            # 8. Headings ## / # (after bold so inner ** still works)
+            s = re.sub(r'^### (.+)$', r'\1', s, flags=re.MULTILINE)
+            s = re.sub(
+                r'^## (.+)$',
+                r'\1',
+                s, flags=re.MULTILINE,
+            )
+            s = re.sub(
+                r'^# (.+)$',
+                r'\1',
+                s, flags=re.MULTILINE,
+            )
+            # 9. Horizontal rules
+            s = re.sub(
+                r'^[ \t]*[-*_]{3,}[ \t]*$',
+                f'
',
+                s, flags=re.MULTILINE,
+            )
+            # 10. Unordered list items (- / * / +) → bullet
+            s = re.sub(r'^[ \t]*[-*+] (.+)$', r'• \1', s, flags=re.MULTILINE)
+            # 11. Numbered list items → bold number
+            s = re.sub(r'^[ \t]*(\d+)\. (.+)$', r'\1. \2', s, flags=re.MULTILINE)
+            # 12. Markdown links [text](url) → text  (not navigable in voice UI)
+            s = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', s)
+            # 13. Bare URLs → dim "[link]" placeholder
+            s = re.sub(
+                r'https?://[^\s\[\](){}<>"&]+',
+                f'[link]',
+                s,
+            )
+            # 14. Blockquotes (after HTML-escape > replaces >)
+            s = re.sub(
+                r'^> (.+)$',
+                rf'\1',
+                s, flags=re.MULTILINE,
+            )
+            # 15. Newlines → 

+            s = s.replace("\n", "
")
+            return s
+
+        def _enforce_block_limit(self):
+            doc = self.conv_view.document()
+            extra = doc.blockCount() - _CONV_MAX_BLOCKS
+            if extra <= 0:
+                return
+            # Remove oldest blocks from the top.
+            cur = QtGui.QTextCursor(doc)
+            cur.movePosition(QtGui.QTextCursor.Start)
+            for _ in range(extra):
+                cur.select(QtGui.QTextCursor.BlockUnderCursor)
+                cur.removeSelectedText()
+                cur.deleteChar()  # remove the trailing newline
+
+        def _bubble_html(self, who: str, text: str) -> str:
+            ts = time.strftime("%H:%M:%S")
+            e = self._esc_html
+
+            if who == "You":
+                # Right-inset card — matches Claude's user message style.
+                return (
+                    f''
+                    f''
+                    f''
+                    f''
+                    f''
+                    f'
'
+                    f'{ts}'
+                    f"  You
"
+                    f'{e(text)}'
+                    f'
\n'
+                )
+
+            if who == "Javis":
+                # Left-aligned with orange "J" badge — Claude's assistant style.
+                return (
+                    f''
+                    f''
+                    f''
+                    f''
+                    f''
+                    f''
+                    f'
'
+                    f'J'
+                    f''
+                    f'Javis'
+                    f' {ts}
'
+                    f'{self._md_to_html(text)}'
+                    f'
\n'
+                )
+
+            if who in ("tool→", "tool←"):
+                direction = "▶" if who == "tool\u2192" else "◀"
+                return (
+                    f''
+                    f''
+                    f''
+                    f''
+                    f'
'
+                    f'{ts}  tool {direction}'
+                    f'  {e(text)}'
+                    f'
\n'
+                )
+
+            if who == "error":
+                return (
+                    f''
+                    f''
+                    f''
+                    f''
+                    f'
'
+                    f'{ts}  {e(text)}'
+                    f'
\n'
+                )
+
+            # system / fallback — centered italic dim line
+            return (
+                f''
+                f''
+                f''
+                f''
+                f'
'
+                f'{ts}  {e(text)}'
+                f'
\n'
+            )
+
+        def _append(self, who: str, text: str):
+            cur = self.conv_view.textCursor()
+            cur.movePosition(QtGui.QTextCursor.End)
+            cur.insertHtml(self._bubble_html(who, text))
+            cur.insertBlock()
+            self.conv_view.setTextCursor(cur)
+            self.conv_view.ensureCursorVisible()
+            self._enforce_block_limit()
+
+        def _begin_assistant_bubble(self):
+            ts = time.strftime("%H:%M:%S")
+            cur = self.conv_view.textCursor()
+            cur.movePosition(QtGui.QTextCursor.End)
+            cur.insertBlock()
+            cur.insertHtml(
+                f'J  '
+                f'Javis '
+                f'{ts} '
+                f''
+            )
+            self.conv_view.setTextCursor(cur)
+            self._assistant_in_progress = True
+            self._assistant_buf = ""
+
+        def _on_assistant_delta(self, delta: str):
+            if not self._assistant_in_progress:
+                self._begin_assistant_bubble()
+            self._assistant_buf += delta
+            cur = self.conv_view.textCursor()
+            cur.movePosition(QtGui.QTextCursor.End)
+            cur.insertText(delta)
+            self.conv_view.setTextCursor(cur)
+            self.conv_view.ensureCursorVisible()
+
+        def _finalize_assistant_bubble(self):
+            if not self._assistant_in_progress:
+                return
+            text = getattr(self, "_assistant_buf", "")
+            self._assistant_in_progress = False
+            cur = self.conv_view.textCursor()
+            cur.movePosition(QtGui.QTextCursor.End)
+            cur.select(QtGui.QTextCursor.BlockUnderCursor)
+            if text.strip():
+                # Atomic in-place replace: swap the streamed plain text for the
+                # markdown-rendered bubble in a single edit so it never flashes
+                # to empty. (insertHtml on a selected range replaces the range.)
+                cur.insertHtml(self._bubble_html("Javis", text))
+            else:
+                # Empty stream — drop the placeholder header block.
+                cur.removeSelectedText()
+                cur.deleteChar()
+            self.conv_view.setTextCursor(cur)
+            self.conv_view.ensureCursorVisible()
+            self._enforce_block_limit()
+
+        def _on_assistant_text(self, text: str):
+            # If we streamed via deltas, just close the open bubble. Otherwise
+            # render as a single bubble (non-streaming fallback path).
+            if self._assistant_in_progress:
+                self._finalize_assistant_bubble()
+            else:
+                self._append("Javis", text)
+
+        def _push_to_talk(self):
+            if not self.orch:
+                return
+            async def _do_ptt():
+                try:
+                    await self.orch.trigger_push_to_talk()
+                except Exception as exc:
+                    log.exception(f"push-to-talk failed: {exc}")
+                    self.errorEvent.emit({"where": "ptt", "message": str(exc)})
+                    if self._assistant_in_progress:
+                        self._finalize_assistant_bubble()
+            asyncio.ensure_future(_do_ptt())
+
+        def _send_text(self):
+            text = self._text_input.text().strip()
+            if not text or not self.orch:
+                return
+            self._last_sent_text = text
+            self._text_input.clear()
+            speak = self._speak_reply_cb.isChecked()
+            async def _do_send():
+                try:
+                    await self.orch.submit_text(text, speak=speak)
+                except Exception as exc:
+                    log.exception(f"submit_text failed: {exc}")
+                    self.errorEvent.emit({"where": "llm", "message": str(exc)})
+                    if self._assistant_in_progress:
+                        self._finalize_assistant_bubble()
+            asyncio.ensure_future(_do_send())
+
+        def _retry_last(self):
+            """Re-submit the last typed or spoken input (Ctrl+R)."""
+            text = getattr(self, "_last_sent_text", "")
+            if not text or not self.orch:
+                return
+            speak = self._speak_reply_cb.isChecked()
+            async def _do_retry():
+                try:
+                    await self.orch.submit_text(text, speak=speak)
+                except Exception as exc:
+                    log.exception(f"retry failed: {exc}")
+                    self.errorEvent.emit({"where": "llm", "message": str(exc)})
+                    if self._assistant_in_progress:
+                        self._finalize_assistant_bubble()
+            asyncio.ensure_future(_do_retry())
+
+        def _interrupt(self):
+            if self.orch:
+                self.orch.interrupt()
+
+        # ──────────────────── system info window ────────────────────────────
+
+        def _show_system_info(self):
+            """Open — or bring to front — the live system information window."""
+            win = getattr(self, "_sys_info_win", None)
+            if win is not None and win.isVisible():
+                win.raise_()
+                win.activateWindow()
+                return
+            self._sys_info_win = self._make_sys_info_window()
+            self._sys_info_win.show()
+
+        def _make_sys_info_window(self):  # noqa: C901
+            import json
+            import os
+            import platform as _platform
+
+            win = QtWidgets.QWidget(self, QtCore.Qt.Window)
+            win.setWindowTitle("System Information — Javis")
+            win.resize(780, 760)
+            win.setWindowFlags(
+                QtCore.Qt.Window
+                | QtCore.Qt.WindowCloseButtonHint
+                | QtCore.Qt.WindowMinimizeButtonHint
+            )
+
+            layout = QtWidgets.QVBoxLayout(win)
+            layout.setContentsMargins(12, 12, 12, 10)
+            layout.setSpacing(8)
+
+            hdr = QtWidgets.QLabel("Javis — Live System Information")
+            self._th(hdr, lambda: f"color:{_CL_TEXT}; font-size:12pt; padding:2px 0;")
+            layout.addWidget(hdr)
+
+            tree = QtWidgets.QTreeWidget()
+            tree.setColumnCount(2)
+            tree.setHeaderLabels(["Property", "Value"])
+            tree.setColumnWidth(0, 290)
+            tree.header().setStretchLastSection(True)
+            tree.setAnimated(False)
+            tree.setIndentation(16)
+            tree.setAlternatingRowColors(True)
+            layout.addWidget(tree, 1)
+
+            footer = QtWidgets.QHBoxLayout()
+            refresh_lbl = QtWidgets.QLabel("Refreshing every second…")
+            self._th(refresh_lbl, lambda: f"color:{_CL_MUTED}; font-size:9pt;")
+            close_btn = QtWidgets.QPushButton("Close")
+            close_btn.setFixedWidth(80)
+            close_btn.clicked.connect(win.close)
+            footer.addWidget(refresh_lbl, 1)
+            footer.addWidget(close_btn)
+            layout.addLayout(footer)
+
+            def _collect_sections():
+                import time as _t
+                sections: list[tuple[str, list[tuple[str, str]]]] = []
+
+                # ── System / Process ─────────────────────────────────────────
+                sys_items: list[tuple[str, str]] = [
+                    ("Python", sys.version.split()[0]),
+                    ("Build", f"{_platform.python_implementation()} {_platform.python_version()}"),
+                    ("Platform", _platform.platform()),
+                    ("Architecture", _platform.machine()),
+                    ("Hostname", _platform.node()),
+                    ("PID", str(os.getpid())),
+                ]
+                try:
+                    import psutil
+                    proc = psutil.Process()
+                    with proc.oneshot():
+                        mi = proc.memory_info()
+                        uptime_s = _t.time() - proc.create_time()
+                        h, r = divmod(int(uptime_s), 3600)
+                        m, s = divmod(r, 60)
+                        sys_items += [
+                            ("Uptime", f"{h}h {m}m {s}s"),
+                            ("RAM (RSS)", f"{mi.rss / 1048576:.1f} MB"),
+                            ("RAM (VMS)", f"{mi.vms / 1048576:.1f} MB"),
+                            ("CPU%", f"{proc.cpu_percent(interval=None):.1f}%"),
+                            ("Threads", str(proc.num_threads())),
+                        ]
+                except ImportError:
+                    sys_items.append(("Resource monitor", "install psutil for RAM/CPU stats"))
+                except Exception as exc:
+                    log.debug(f"failed to collect psutil process metrics: {exc}")
+                    sys_items.append(("psutil error", str(exc)))
+                sections.append(("🖥  System", sys_items))
+
+                # ── Javis Config ─────────────────────────────────────────────
+                cfg_items: list[tuple[str, str]] = [
+                    ("Config file", str(config.source_path or "")),
+                    ("Data directory", str(config.paths.base_dir)),
+                    ("Preset", config.agent.preset),
+                    ("History max turns", str(config.agent.history_max_turns)),
+                    ("Max LLM iterations", str(config.agent.max_llm_iterations)),
+                    ("MCP approval default", config.mcp.approval_default),
+                    ("MCP call timeout", f"{config.mcp.call_timeout_seconds} s"),
+                ]
+                sections.append(("⚙  Config", cfg_items))
+
+                # ── LLM ──────────────────────────────────────────────────────
+                llm_items: list[tuple[str, str]] = [
+                    ("URL", config.llm.base_url),
+                    ("Model", config.llm.model),
+                    ("Context window", f"{config.llm.num_ctx:,} tokens"),
+                    ("Temperature", str(config.llm.temperature)),
+                    ("Request timeout", f"{config.llm.request_timeout_seconds} s"),
+                ]
+                sections.append(("🤖  LLM (Ollama)", llm_items))
+
+                # ── STT ──────────────────────────────────────────────────────
+                if config.stt.enabled:
+                    stt_items: list[tuple[str, str]] = [
+                        ("Status", "enabled"),
+                        ("Model", config.stt.model_size),
+                        ("Device", config.stt.device),
+                        ("Compute type", config.stt.compute_type),
+                        ("Beam size", str(config.stt.beam_size)),
+                        ("Language", config.stt.language or "auto-detect"),
+                        ("Condition on prev.", str(config.stt.condition_on_previous_text)),
+                    ]
+                else:
+                    stt_items = [("Status", "disabled — text-only mode")]
+                sections.append(("🎤  Speech-to-Text", stt_items))
+
+                # ── TTS ──────────────────────────────────────────────────────
+                if config.tts.enabled:
+                    tts_items: list[tuple[str, str]] = [
+                        ("Status", "enabled"),
+                        ("Backend", config.tts.backend),
+                        ("Default voice", config.tts.default_voice),
+                        ("Device", config.tts.device),
+                    ]
+                else:
+                    tts_items = [("Status", "disabled")]
+                sections.append(("🔊  Text-to-Speech", tts_items))
+
+                # ── Audio ─────────────────────────────────────────────────────
+                aud_items: list[tuple[str, str]] = [
+                    ("Sample rate", f"{config.audio.sample_rate} Hz"),
+                    ("Input device", str(config.audio.input_device or "system default")),
+                    ("Output device", str(config.audio.output_device or "system default")),
+                ]
+                sections.append(("🎵  Audio", aud_items))
+
+                # ── Wake Word ─────────────────────────────────────────────────
+                if config.wake_word.enabled:
+                    ww_items: list[tuple[str, str]] = [
+                        ("Status", "enabled"),
+                        ("Model", config.wake_word.model),
+                        ("Threshold", f"{config.wake_word.threshold:.0%}"),
+                        ("VAD threshold", f"{config.wake_word.vad_threshold:.0%}"),
+                        ("Silence", f"{config.wake_word.silence_ms} ms"),
+                    ]
+                else:
+                    ww_items = [("Status", "disabled")]
+                sections.append(("👂  Wake Word", ww_items))
+
+                # ── Live agent state ──────────────────────────────────────────
+                orch = self.orch
+                if orch is not None:
+                    try:
+                        state_val = orch.state.value if hasattr(orch.state, "value") else str(orch.state)
+                        history_len = len(orch.conv.history)
+                        sys_prompt_len = len(orch.conv.system_prompt)
+                        approx_tok = orch.conv.approx_tokens
+                        agent_items: list[tuple[str, str]] = [
+                            ("State", state_val),
+                            ("History messages", str(history_len)),
+                            ("Approx. tokens", f"{approx_tok:,}"),
+                            ("System prompt chars", f"{sys_prompt_len:,}"),
+                        ]
+                        sections.append(("🔄  Agent (live)", agent_items))
+                    except Exception as exc:
+                        log.debug(f"failed to collect live agent information: {exc}")
+                        sections.append(("🔄  Agent (live)", [("error", str(exc))]))
+
+                    # ── MCP Servers ───────────────────────────────────────────
+                    try:
+                        snap = orch.mcp.status_snapshot()
+                        servers = snap.get("servers", [])
+                        mcp_items: list[tuple[str, str]] = [
+                            ("Server count", str(len(servers))),
+                            ("Total tools", str(snap.get("tool_count", 0))),
+                        ]
+                        for srv in servers:
+                            tools = srv.get("tools", [])
+                            preview = ", ".join(tools[:5])
+                            if len(tools) > 5:
+                                preview += f"  … +{len(tools) - 5} more"
+                            mcp_items.append((
+                                f"  ↳ {srv.get('name', '?')}",
+                                f"{len(tools)} tools  [{srv.get('approval', '?')}]  {preview}",
+                            ))
+                        sections.append(("🔌  MCP Servers (live)", mcp_items))
+                    except Exception as exc:
+                        log.debug(f"failed to collect MCP server information: {exc}")
+                        sections.append(("🔌  MCP Servers (live)", [("error", str(exc))]))
+
+                    # ── Middleware stack ──────────────────────────────────────
+                    try:
+                        mw_list = orch.middleware
+                        mw_items: list[tuple[str, str]] = [
+                            ("Count", str(len(mw_list)))
+                        ]
+                        for idx, mw in enumerate(mw_list):
+                            mw_items.append((
+                                f"  [{idx}]",
+                                getattr(mw, "name", type(mw).__name__),
+                            ))
+                        sections.append(("🧩  Middleware (live)", mw_items))
+                    except Exception as exc:
+                        log.debug(f"failed to collect middleware information: {exc}")
+                        sections.append(("🧩  Middleware (live)", [("error", str(exc))]))
+                else:
+                    sections.append(("🔄  Agent", [("Status", "starting up…")]))
+
+                # ── Memory Store ──────────────────────────────────────────────
+                mem_file = config.paths.base_dir / "memory.json"
+                try:
+                    raw = mem_file.read_text(encoding="utf-8")
+                    data = json.loads(raw)
+                    entries = data.get("entries", {})
+                    cats: dict[str, int] = {}
+                    for e in entries.values():
+                        cat = e.get("category", "other")
+                        cats[cat] = cats.get(cat, 0) + 1
+                    mem_items: list[tuple[str, str]] = [
+                        ("File", str(mem_file)),
+                        ("File size", f"{len(raw.encode()) / 1024:.1f} KB"),
+                        ("Entry count", str(len(entries))),
+                        ("Categories", "  ".join(f"{k}:{v}" for k, v in cats.items()) or "—"),
+                    ]
+                    for key, entry in list(entries.items())[-8:]:
+                        val_preview = str(entry.get("value", ""))[:72]
+                        cat = entry.get("category", "?")
+                        mem_items.append((f"  {key}", f"[{cat}]  {val_preview}"))
+                except FileNotFoundError:
+                    mem_items = [
+                        ("File", str(mem_file)),
+                        ("Status", "not found — no memories stored yet"),
+                    ]
+                except Exception as exc:
+                    log.debug(f"failed to collect memory store information: {exc}")
+                    mem_items = [("File", str(mem_file)), ("Error", str(exc))]
+                sections.append(("🧠  Memory Store (live)", mem_items))
+
+                return sections
+
+            def _refresh():
+                import time as _t
+                sections = _collect_sections()
+
+                # Preserve which sections are currently expanded.
+                expanded: set[str] = set()
+                for i in range(tree.topLevelItemCount()):
+                    item = tree.topLevelItem(i)
+                    if item.isExpanded():
+                        expanded.add(item.text(0))
+                first_fill = tree.topLevelItemCount() == 0
+
+                tree.setUpdatesEnabled(False)
+                tree.clear()
+                bold = QtGui.QFont("Segoe UI", 9, QtGui.QFont.Bold)
+                for title, children in sections:
+                    section = QtWidgets.QTreeWidgetItem([title, ""])
+                    section.setFont(0, bold)
+                    section.setForeground(0, QtGui.QColor(_CL_ACCENT))
+                    for key, val in children:
+                        child = QtWidgets.QTreeWidgetItem([key, val])
+                        child.setForeground(0, QtGui.QColor(_CL_MUTED))
+                        child.setForeground(1, QtGui.QColor(_CL_TEXT))
+                        section.addChild(child)
+                    tree.addTopLevelItem(section)
+                    # Expand all on first open; restore state on subsequent refreshes.
+                    section.setExpanded(first_fill or title in expanded)
+                tree.setUpdatesEnabled(True)
+                refresh_lbl.setText(f"Last updated: {_t.strftime('%H:%M:%S')}  —  auto-refresh 1 s")
+
+            _refresh()
+
+            timer = QtCore.QTimer(win)
+            timer.setInterval(1000)
+            timer.timeout.connect(_refresh)
+            timer.start()
+            win.destroyed.connect(timer.stop)
+
+            return win
+
+    # --- app + loop setup ---
+    app = QtWidgets.QApplication.instance() or QtWidgets.QApplication(sys.argv)
+
+    # ── theme: rebind the _CL_* tokens to the active palette so every
+    #    function in this scope (bubbles, badges, approval dialog, handlers)
+    #    picks up the current colors. Defined here so the startup dialog is
+    #    already themed before it opens. ───────────────────────────────────────
+    def _derive_tokens(pal: dict):
+        global _CL_BG, _CL_SURFACE, _CL_SURFACE2, _CL_INPUT, _CL_ACCENT, \
+            _CL_ACCENT2, _CL_TEXT, _CL_MUTED, _CL_BORDER, _CL_BORDER2, \
+            _CL_PURPLE, _CL_SEL, _CL_SUCCESS, _CL_WARN, _CL_ERROR, _CL_ERROR_BG, \
+            _CL_ERROR_BORDER, _CL_SUCCESS_BG, _CL_SUCCESS_BORDER, _CL_WARN_FG, \
+            _CL_CODE_BG, _CL_CODE_FG, _CL_TOOL_BG, _CL_TOOL_FG, _CL_TOOL_MUTED, \
+            _CL_HR, _CL_LINK, _CL_QUOTE, _CL_USER_NAME
+        _CL_BG, _CL_SURFACE, _CL_SURFACE2, _CL_INPUT = pal["bg"], pal["surface"], pal["surface2"], pal["input"]
+        _CL_ACCENT, _CL_ACCENT2 = pal["accent"], pal["accent2"]
+        _CL_TEXT, _CL_MUTED, _CL_BORDER, _CL_BORDER2 = pal["text"], pal["muted"], pal["border"], pal["border2"]
+        _CL_PURPLE, _CL_SEL = pal["purple"], pal["sel"]
+        _CL_SUCCESS, _CL_WARN = pal["success"], pal["warn"]
+        _CL_ERROR, _CL_ERROR_BG = pal["error"], pal["error_bg"]
+        _CL_ERROR_BORDER = pal["error_border"]
+        _CL_SUCCESS_BG, _CL_SUCCESS_BORDER = pal["success_bg"], pal["success_border"]
+        _CL_WARN_FG = pal["warn_fg"]
+        _CL_CODE_BG, _CL_CODE_FG = pal["code_bg"], pal["code_fg"]
+        _CL_TOOL_BG, _CL_TOOL_FG, _CL_TOOL_MUTED = pal["tool_bg"], pal["tool_fg"], pal["tool_muted"]
+        _CL_HR, _CL_LINK, _CL_QUOTE, _CL_USER_NAME = pal["hr"], pal["link"], pal["quote"], pal["user_name"]
+        _STATE_COLORS.clear()
+        _STATE_COLORS.update({
+            "idle": pal["state_idle"], "wake": pal["purple"], "listening": pal["success"],
+            "thinking": pal["accent"], "speaking": pal["success"], "error": pal["error"],
+            "enrolling": pal["purple"],
+        })
+
+    def _apply_theme(name: str):
+        pal = _THEME_NAMES.get(name, _PAL_DARK)
+        _derive_tokens(pal)
+        app.setStyleSheet(_build_app_qss(pal))
+        app._javis_theme = name
+        if getattr(config.gui, "theme", "dark") != name:
+            config.gui.theme = name
+        # The main window is NOT yet constructed when this runs at startup
+        # (the theme must be applied before _MainWindow() so its widgets pick
+        # up the right _CL_* tokens). Guard the window-specific refresh so the
+        # early call simply skips it.
+        try:
+            win = window
+        except NameError:
+            win = None
+        if win is not None:
+            # Re-apply all registered per-widget inline styles to the new theme
+            # (they bake in token values, so they override the app stylesheet).
+            if hasattr(win, "retheme"):
+                win.retheme()
+            if hasattr(win, "state_badge"):
+                win.state_badge.setStyleSheet(win._badge_style(win.state_badge.text().strip()))
+                if hasattr(win, "state_dot"):
+                    win.state_dot.setStyleSheet(win._dot_style("idle"))
+            # Keep the Tools→toggle menu action in sync (block signals to avoid recursion).
+            if hasattr(win, "_theme_act"):
+                win._theme_act.blockSignals(True)
+                win._theme_act.setChecked(name == "light")
+                win._theme_act.blockSignals(False)
+
+    # Ask which TTS model to load BEFORE we spin up the asyncio loop and
+    # orchestrator, so the user can avoid waiting on a big model load.
+    # Skippable via gui.show_startup_dialog (Tools → Startup configuration… reopens it).
+    if config.gui.show_startup_dialog:
+        picked = show_startup_dialog(config)
+        if picked is None:
+            log.info("startup dialog cancelled — exiting")
+            return 0
+        config = picked
+
+    # Apply the persisted theme (default dark) now that config is final.
+    _apply_theme(getattr(config.gui, "theme", "dark") or "dark")
+
+    # The main window hides to the tray; closing a dialog while it is hidden
+    # must not terminate the app.  Quit is explicit (File → Quit / tray menu).
+    app.setQuitOnLastWindowClosed(False)
+
+    loop = qasync.QEventLoop(app)
+    asyncio.set_event_loop(loop)
+
+    window = _MainWindow()
+    window.show()
+
+    from .hotkey import install_global_hotkey
+    # Keep a reference: Qt holds native event filters weakly.
+    app._javis_hotkey = install_global_hotkey(config.gui.global_ptt_hotkey, window._push_to_talk)
+
+    bus = EventBus()
+    orch = Orchestrator(config, bus=bus)
+    window.orch = orch
+
+    # ── slave-device API (javis-slave companions) ────────────────────────────
+    from ..api.devices import SlaveDeviceStore
+    from ..api.pairing import PairingCodeManager
+    window.device_store = SlaveDeviceStore(config.paths.slaves_file)
+    window.pairing_mgr = PairingCodeManager(config.api.pairing_code_ttl_seconds)
+
+    async def _on_state(s):
+        window.stateChanged.emit(s.value if hasattr(s, "value") else str(s))
+    async def _on_user(t):    window.userText.emit(str(t))
+    async def _on_assi(t):    window.assistantText.emit(str(t))
+    async def _on_assi_delta(d): window.assistantDelta.emit(str(d))
+    async def _on_tool(p):    window.toolCall.emit(str(p.get("name", "?")), str(p.get("args", {})))
+    async def _on_tool_result(p): window.toolResult.emit(p or {})
+    async def _on_stt(p):     window.sttDone.emit(p or {})
+    async def _on_llm(p):     window.llmStatus.emit(p or {})
+    async def _on_tts(p):     window.ttsStatus.emit(p or {})
+    async def _on_mcp(p):     window.mcpReady.emit(p or {})
+    async def _on_health(p):  window.llmHealth.emit(p or {})
+    async def _on_level(p):   window.micLevel.emit(p or {})
+    async def _on_wake(p):    window.wakeFired.emit(float(p) if p is not None else 0.0)
+    async def _on_ingest(p):  window.ingestStatus.emit(p or {})
+    async def _on_error(p):   window.errorEvent.emit(p or {})
+    async def _on_notice(p):  window.noticeEvent.emit(p or {})
+    async def _on_task_start(p): window.taskStart.emit(p or {})
+    async def _on_task_done(p):  window.taskDone.emit(p or {})
+    async def _on_quick_match_ev(p): window.quickMatch.emit(p or {})
+    async def _on_quick_updated_ev(_p): window.quickUpdated.emit()
+    async def _on_speaker_id_ev(p): window.speakerIdentified.emit(p or {})
+    async def _on_speaker_enrolled_ev(p): window.speakerEnrolled.emit(p or {})
+    async def _on_speaker_ask_name_ev(p): window.speakerAskName.emit(p or {})
+    async def _on_enroll_progress_ev(p): window.enrollProgress.emit(p or {})
+    async def _on_timer_fired_ev(p):     window.timerFired.emit(p or {})
+
+    bus.subscribe("state", _on_state)
+    bus.subscribe("user_text", _on_user)
+    bus.subscribe("assistant_text", _on_assi)
+    bus.subscribe("assistant_delta", _on_assi_delta)
+    bus.subscribe("tool_call", _on_tool)
+    bus.subscribe("tool_result", _on_tool_result)
+    bus.subscribe("stt_done", _on_stt)
+    bus.subscribe("llm_status", _on_llm)
+    bus.subscribe("tts_status", _on_tts)
+    bus.subscribe("mcp_ready", _on_mcp)
+    bus.subscribe("llm_health", _on_health)
+    bus.subscribe("mic_level", _on_level)
+    bus.subscribe("wake", _on_wake)
+    bus.subscribe("ingest_status", _on_ingest)
+    bus.subscribe("error", _on_error)
+    bus.subscribe("notice", _on_notice)
+    bus.subscribe("task_start", _on_task_start)
+    bus.subscribe("task_done", _on_task_done)
+    bus.subscribe("quick_match", _on_quick_match_ev)
+    bus.subscribe("quick_updated", _on_quick_updated_ev)
+    bus.subscribe("speaker_id", _on_speaker_id_ev)
+    bus.subscribe("speaker_enrolled", _on_speaker_enrolled_ev)
+    bus.subscribe("speaker_ask_name", _on_speaker_ask_name_ev)
+    bus.subscribe("enroll_progress", _on_enroll_progress_ev)
+    bus.subscribe("timer_fired", _on_timer_fired_ev)
+
+    # MCP approval dialog
+    class _ApprovalDialog(QtWidgets.QDialog):
+        """Themed MCP tool-approval dialog (replaces the bare QMessageBox).
+
+        Exposes ``result_value`` in {"once", "always", "deny"} after ``exec()``.
+        Accent primary "Allow once", neutral "Always allow", red "Deny".
+        """
+
+        def __init__(self, server: str, tool: str, details: str):
+            super().__init__(window)
+            self.setWindowTitle("MCP tool approval")
+            self.setMinimumWidth(440)
+            self.result_value: str = "deny"
+
+            lay = QtWidgets.QVBoxLayout(self)
+            lay.setContentsMargins(18, 18, 18, 18)
+            lay.setSpacing(10)
+
+            title = QtWidgets.QLabel("Allow tool call?")
+            title.setStyleSheet(f"font-size: 11pt; color: {_CL_TEXT};")
+            lay.addWidget(title)
+
+            meta = QtWidgets.QLabel(
+                f"Server: {server}
"
+                f"Tool:   {tool}"
+            )
+            meta.setTextFormat(QtCore.RichText)
+            meta.setStyleSheet(f"color: {_CL_MUTED}; padding: 4px;")
+            lay.addWidget(meta)
+
+            args_box = QtWidgets.QPlainTextEdit()
+            args_box.setReadOnly(True)
+            args_box.setPlainText(details)
+            args_box.setFixedHeight(110)
+            args_box.setStyleSheet(
+                f"background: {_CL_BG}; color: {_CL_TEXT}; border: 1px solid {_CL_BORDER}; "
+                f"border-radius: 6px; padding: 6px; font-family: Consolas, monospace; font-size: 9pt;")
+            lay.addWidget(args_box)
+
+            if config.mcp.voice_approval and orch.stt is not None and orch.mic is not None:
+                hint = QtWidgets.QLabel("🎤 You can also answer aloud: „ja“ / „nein“")
+                hint.setStyleSheet(f"color: {_CL_MUTED}; font-size: 9pt;")
+                lay.addWidget(hint)
+
+            btns = QtWidgets.QHBoxLayout()
+            btns.setSpacing(10)
+            deny = QtWidgets.QPushButton("Deny")
+            always = QtWidgets.QPushButton("Always allow")
+            once = QtWidgets.QPushButton("Allow once")
+            once.setDefault(True)
+            once.setStyleSheet(
+                f"background: {_CL_ACCENT}; color: #1e1e1e; font-weight: 600; "
+                f"padding: 8px 16px; border-radius: 6px;")
+            always.setStyleSheet(
+                f"background: {_CL_SURFACE2}; color: {_CL_TEXT}; "
+                f"padding: 8px 16px; border-radius: 6px; border: 1px solid {_CL_BORDER2};")
+            deny.setStyleSheet(
+                f"background: {_CL_ERROR_BG}; color: {_CL_ERROR}; "
+                f"padding: 8px 16px; border-radius: 6px; border: 1px solid {_CL_ERROR};")
+            btns.addStretch(1)
+            btns.addWidget(deny)
+            btns.addWidget(always)
+            btns.addWidget(once)
+            lay.addLayout(btns)
+
+            def _result(value: str):
+                self.result_value = value
+                self.accept()
+
+            once.clicked.connect(lambda: _result("once"))
+            always.clicked.connect(lambda: _result("always"))
+            deny.clicked.connect(lambda: _result("deny"))
+
+    async def _approval(server: str, tool: str, args: dict) -> bool:
+        # Auto-approve skips the dialog entirely.
+        if window._auto_approve_btn.isChecked():
+            return True
+
+        future: asyncio.Future = asyncio.get_running_loop().create_future()
+        holder: dict = {"dlg": None}
+        t_start = time.monotonic()
+
+        def _ask():
+            details = str(args)
+            if len(details) > 400:
+                details = details[:400] + "…"
+            dlg = _ApprovalDialog(server, tool, details)
+            holder["dlg"] = dlg
+            # Bring the window forward so a voice user can find the prompt.
+            if not window.isVisible() or window.isMinimized():
+                window._show_from_tray()
+            dlg.exec()
+            result = dlg.result_value
+            if not future.done():
+                if result == "always":
+                    orch.mcp.allow_tool(server, tool)
+                    future.set_result(True)
+                elif result == "once":
+                    future.set_result(True)
+                else:
+                    future.set_result(False)
+
+        # Audible cue + optional spoken yes/no answer, racing the dialog.
+        speak = not window.isActiveWindow()
+        timeout = config.mcp.approval_timeout_seconds
+        voice_task: asyncio.Task | None = None
+        if config.mcp.voice_approval and orch.stt is not None and orch.mic is not None:
+            voice_task = asyncio.create_task(
+                orch.approval_by_voice(timeout if timeout > 0 else 30.0, speak=speak))
+        else:
+            asyncio.ensure_future(orch.notify_approval_pending(speak=speak))
+        QtCore.QTimer.singleShot(0, _ask)
+
+        def _close_dialog() -> None:
+            dlg = holder["dlg"]
+            if dlg is not None:
+                dlg.reject()
+
+        try:
+            waiters = {future} | ({voice_task} if voice_task else set())
+            while True:
+                left = None
+                if timeout > 0:
+                    left = timeout - (time.monotonic() - t_start)
+                    if left <= 0:
+                        raise asyncio.TimeoutError
+                done, _ = await asyncio.wait(waiters, timeout=left,
+                                             return_when=asyncio.FIRST_COMPLETED)
+                if not done:
+                    raise asyncio.TimeoutError
+                if future in done:
+                    return future.result()
+                # Voice task finished: a real answer decides, None keeps waiting.
+                answer = voice_task.result() if voice_task and not voice_task.cancelled() else None
+                waiters.discard(voice_task)
+                voice_task = None
+                if answer is not None:
+                    # Settle first so _ask() sees future.done() when exec() returns.
+                    if not future.done():
+                        future.set_result(bool(answer))
+                    _close_dialog()
+                    window.noticeEvent.emit({
+                        "where": "mcp",
+                        "message": f"Tool '{tool}' {'approved' if answer else 'denied'} by voice."})
+                    return bool(answer)
+                if not waiters:
+                    return False
+        except asyncio.TimeoutError:
+            if not future.done():
+                future.set_result(False)
+            _close_dialog()
+            window.noticeEvent.emit({
+                "where": "mcp",
+                "message": f"Tool '{tool}' denied — no approval within {timeout}s."})
+            return False
+        finally:
+            if voice_task is not None and not voice_task.done():
+                voice_task.cancel()
+
+    orch.mcp.approval_cb = _approval
+
+    async def _bootstrap():
+        try:
+            await orch.start()
+            
+            # Fetch available models from Ollama and populate the combo box
+            try:
+                if orch.llm is not None:
+                    ollama_models = await orch.llm.list_models()
+                    model_names = ollama_models if ollama_models else []
+                    # Update UI on main thread using QTimer
+                    QtCore.QTimer.singleShot(0, lambda: window._populate_model_combo(model_names))
+            except Exception as e:
+                log.warning(f"Failed to fetch Ollama models for combo box: {e}")
+            
+            asyncio.create_task(orch.run())
+
+            # Overnight voice training (resumes from latest checkpoint).
+            # Always started; the scheduler itself polls config.training.enabled
+            # each cycle so it can be toggled live without a restart.
+            from ..voices.training_scheduler import training_scheduler
+            asyncio.create_task(training_scheduler(config, bus=bus))
+
+            if config.api.enabled:
+                from ..api.server import build_app, run_server
+                api_app = build_app(config, orch, window.device_store, window.pairing_mgr)
+                asyncio.create_task(run_server(config, api_app))
+        except Exception as e:
+            log.exception(f"startup failed: {e}")
+            QtWidgets.QMessageBox.critical(window, "Startup failed", str(e))
+
+    asyncio.ensure_future(_bootstrap())
+
+    async def _shutdown():
+        try:
+            await orch.stop()
+        except Exception:
+            pass
+
+    with loop:
+        loop.run_forever()
+        hk = getattr(app, "_javis_hotkey", None)
+        if hk is not None:
+            try:
+                hk.uninstall()
+            except Exception:  # noqa: BLE001
+                pass
+        # The Qt app has quit and the loop has stopped (but is not yet closed).
+        # Run the async shutdown to completion so orch.stop() actually finishes
+        # instead of being lost as a fire-and-forget task.
+        try:
+            loop.run_until_complete(_shutdown())
+        except Exception:
+            log.exception("shutdown failed")
+    return 0
diff --git a/src/javis/gui/startup_dialog.py b/src/javis/gui/startup_dialog.py
new file mode 100644
index 0000000..5d0f972
--- /dev/null
+++ b/src/javis/gui/startup_dialog.py
@@ -0,0 +1,1043 @@
+"""Startup configuration dialog.
+
+Shown before the main window so the user can review and edit all Javis settings,
+choose which STT / TTS models to load, and clean up corrupt / old downloads.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from ..core.config import Config
+from ..core.config_io import update_config_file
+from ..agent import prompts as agent_prompts
+from .theme import palette_for
+
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# Whisper model catalog: (size_id, display_label, approx_disk_size)
+_WHISPER_SIZES: tuple[tuple[str, str, str], ...] = (
+    ("tiny",              "tiny — fastest, lowest quality",                      "~75 MB"),
+    ("base",              "base — fast",                                          "~145 MB"),
+    ("small",             "small — balanced",                                     "~480 MB"),
+    ("medium",            "medium — accurate",                                    "~1.5 GB"),
+    ("large-v3",          "large-v3 — best quality",                             "~3.1 GB"),
+    ("large-v3-turbo",    "large-v3-turbo — large quality, ~8x faster",          "~1.6 GB"),
+    ("distil-large-v3.5", "distil-large-v3.5 — improved distil, fast",          "~1.5 GB"),
+    ("distil-large-v3",   "distil-large-v3 — large quality, ~2x faster",        "~1.5 GB"),
+)
+
+def _dialog_qss(t: dict) -> str:
+    """Render the dialog stylesheet for a theme palette (see gui.theme)."""
+    return f"""
+QDialog {{ background:{t['bg']}; color:{t['text']}; }}
+QWidget {{ color:{t['text']}; font-family:'Segoe UI','Inter',sans-serif; font-size:10pt; }}
+QFrame#Header {{ background:{t['surface']}; border-bottom:1px solid {t['border']}; }}
+QFrame#Footer {{ background:{t['surface']}; border-top:1px solid {t['border']}; }}
+QLabel#Title {{ font-size:18pt; font-weight:600; color:{t['text']}; }}
+QLabel#Subtitle {{ color:{t['muted']}; font-size:10pt; }}
+QLabel#Hint {{ color:{t['muted']}; font-size:9pt; }}
+QScrollArea {{ background:{t['bg']}; border:0; }}
+QGroupBox {{
+    background:{t['surface']}; border:1px solid {t['border']}; border-radius:6px;
+    margin-top:14px; padding:12px 10px 10px 10px;
+}}
+QGroupBox::title {{
+    subcontrol-origin:margin; left:12px; padding:0 6px;
+    color:{t['text']}; font-weight:600;
+}}
+QPushButton {{
+    background:{t['surface']}; color:{t['text']}; border:1px solid {t['border']};
+    padding:6px 14px; border-radius:4px;
+}}
+QPushButton:hover {{ background:{t['btn_hover']}; border-color:{t['border2']}; }}
+QPushButton:pressed {{ background:{t['btn_pressed']}; }}
+QPushButton#Primary {{ background:{t['accent']}; border-color:{t['accent']}; color:white; font-weight:600; }}
+QPushButton#Primary:hover {{ background:{t['accent2']}; border-color:{t['accent2']}; }}
+QRadioButton, QCheckBox {{ spacing:8px; color:{t['text']}; }}
+QComboBox, QLineEdit, QSpinBox, QDoubleSpinBox {{
+    background:{t['input']}; color:{t['text']}; border:1px solid {t['border']};
+    padding:4px 6px; border-radius:4px;
+}}
+QComboBox:focus, QLineEdit:focus, QSpinBox:focus, QDoubleSpinBox:focus {{ border-color:{t['accent']}; }}
+QComboBox QAbstractItemView {{
+    background:{t['surface']}; color:{t['text']}; border:1px solid {t['border']};
+    selection-background-color:{t['sel']};
+}}
+QListWidget, QTableWidget {{
+    background:{t['surface']}; color:{t['text']}; border:1px solid {t['border']}; border-radius:4px;
+}}
+QListWidget::item:selected, QTableWidget::item:selected {{ background:{t['sel']}; color:{t['text']}; }}
+QListWidget::item:hover, QTableWidget::item:hover {{ background:{t['item_hover']}; }}
+QHeaderView::section {{
+    background:{t['surface']}; color:{t['muted']}; padding:4px 8px;
+    border:0; border-bottom:1px solid {t['border']};
+}}
+QProgressBar {{ background:{t['input']}; border:1px solid {t['border']}; border-radius:4px;
+                text-align:center; color:{t['text']}; }}
+QProgressBar::chunk {{ background:{t['success']}; border-radius:3px; }}
+QScrollBar:vertical {{ background:{t['bg']}; width:10px; border:0; }}
+QScrollBar::handle:vertical {{ background:{t['handle']}; border-radius:5px; min-height:20px; }}
+QScrollBar::handle:vertical:hover {{ background:{t['handle_hover']}; }}
+QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical {{ height:0; }}
+QScrollBar:horizontal {{ background:{t['bg']}; height:10px; border:0; }}
+QScrollBar::handle:horizontal {{ background:{t['handle']}; border-radius:5px; }}
+QScrollBar::add-line:horizontal, QScrollBar::sub-line:horizontal {{ width:0; }}
+"""
+
+
+# ---------------------------------------------------------------------------
+# STT helpers
+# ---------------------------------------------------------------------------
+
+def _whisper_cache_dir(config: Config) -> Path:
+    return config.paths.models_dir / "whisper"
+
+
+def _whisper_cached_sizes(config: Config) -> set[str]:
+    """Return set of Whisper size_ids that are already on disk."""
+    cache_root = _whisper_cache_dir(config)
+    if not cache_root.exists():
+        return set()
+    found: set[str] = set()
+    try:
+        for child in cache_root.rglob("*"):
+            if not child.is_dir():
+                continue
+            name = child.name.lower()
+            for size_id, _, _ in _WHISPER_SIZES:
+                if size_id.lower() in name:
+                    found.add(size_id)
+    except OSError:
+        pass
+    return found
+
+
+# ---------------------------------------------------------------------------
+# Voice-profile helpers
+# ---------------------------------------------------------------------------
+
+def _dir_size_bytes(path: Path) -> int:
+    """Return total byte size of every file under *path*."""
+    total = 0
+    try:
+        for root, _, files in os.walk(path):
+            for fname in files:
+                try:
+                    total += (Path(root) / fname).stat().st_size
+                except OSError:
+                    pass
+    except OSError:
+        pass
+    return total
+
+
+def _format_bytes(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024:
+            return f"{n:.1f} {unit}"
+        n /= 1024
+    return f"{n:.1f} TB"
+
+
+def _scan_voice_profiles(voices_dir: Path) -> list[dict]:
+    """Scan *voices_dir* and return one dict per profile.
+
+    Keys: name, path, size_bytes, ref_count, train_count, corrupt, reasons.
+    """
+    profiles: list[dict] = []
+    if not voices_dir.exists():
+        return profiles
+    for profile_dir in sorted(voices_dir.iterdir()):
+        if not profile_dir.is_dir():
+            continue
+        refs_dir   = profile_dir / "refs"
+        train_dir  = profile_dir / "training" / "wavs"
+        meta_file  = profile_dir / "profile.json"
+        ref_clips   = list(refs_dir.glob("*.wav"))   if refs_dir.exists()  else []
+        train_clips = list(train_dir.glob("*.wav")) if train_dir.exists() else []
+        corrupt_reasons: list[str] = []
+        if not meta_file.exists():
+            corrupt_reasons.append("no profile.json")
+        if not ref_clips:
+            corrupt_reasons.append("no reference clips")
+        profiles.append({
+            "name":        profile_dir.name,
+            "path":        profile_dir,
+            "size_bytes":  _dir_size_bytes(profile_dir),
+            "ref_count":   len(ref_clips),
+            "train_count": len(train_clips),
+            "corrupt":     bool(corrupt_reasons),
+            "reasons":     corrupt_reasons,
+        })
+    return profiles
+
+
+# ---------------------------------------------------------------------------
+# YAML persistence
+# ---------------------------------------------------------------------------
+
+def _save_config_to_yaml(config: Config) -> None:
+    """Merge all editable config fields into config.source_path (atomic, comment-preserving).
+
+    Raises on failure so the caller can tell the user the save did not happen.
+    """
+    path = config.source_path
+    if path is None:
+        raise RuntimeError("No config.yaml was loaded — start Javis with a config file to persist settings.")
+    update_config_file(path, {
+        "llm": {
+            "base_url":                config.llm.base_url,
+            "model":                   config.llm.model,
+            "temperature":             config.llm.temperature,
+            "num_ctx":                 config.llm.num_ctx,
+            "request_timeout_seconds": config.llm.request_timeout_seconds,
+        },
+        "stt": {
+            "enabled":    config.stt.enabled,
+            "model_size": config.stt.model_size,
+        },
+        "tts": {
+            "enabled": config.tts.enabled,
+            "backend": config.tts.backend,
+        },
+        "agent": {
+            "preset":               config.agent.preset,
+            "system_prompt_inline": config.agent.system_prompt_inline,
+        },
+        "audio": {
+            "input_device":  config.audio.input_device,
+            "output_device": config.audio.output_device,
+            "input_gain":    config.audio.input_gain,
+        },
+        "wake_word": {
+            "enabled":       config.wake_word.enabled,
+            "model":         config.wake_word.model,
+            "threshold":     config.wake_word.threshold,
+            "vad_threshold": config.wake_word.vad_threshold,
+            "silence_ms":    config.wake_word.silence_ms,
+        },
+        "speaker_id": {
+            "enabled":          config.speaker_id.enabled,
+            "threshold":        config.speaker_id.threshold,
+            "min_enroll_clips": config.speaker_id.min_enroll_clips,
+            "auto_enroll":      config.speaker_id.auto_enroll,
+        },
+        "memory": {
+            "model":           config.memory.model,
+            "temperature":     config.memory.temperature,
+            "num_ctx":         config.memory.num_ctx,
+            "timeout_seconds": config.memory.timeout_seconds,
+        },
+        "gui": {
+            "show_startup_dialog": config.gui.show_startup_dialog,
+        },
+    })
+
+
+# ---------------------------------------------------------------------------
+# Section builders  —  each returns (QGroupBox, widgets_dict)
+# ---------------------------------------------------------------------------
+
+def _build_llm_section(config: Config) -> tuple:
+    """LLM / Ollama connection and generation settings."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Language Model  (LLM)")
+    form = QtWidgets.QFormLayout(box)
+    form.setVerticalSpacing(8)
+    form.setContentsMargins(10, 16, 10, 10)
+
+    url_edit = QtWidgets.QLineEdit(config.llm.base_url or "http://localhost:11434")
+    url_edit.setPlaceholderText("http://localhost:11434")
+    url_edit.setToolTip("Ollama API base URL")
+    form.addRow("Ollama URL:", url_edit)
+
+    model_edit = QtWidgets.QLineEdit(config.llm.model or "qwen2.5:7b")
+    model_edit.setPlaceholderText("e.g. qwen2.5:7b, llama3.1:8b")
+    model_edit.setToolTip("Model name as listed in 'ollama list'")
+    form.addRow("Model:", model_edit)
+
+    temperature_spin = QtWidgets.QDoubleSpinBox()
+    temperature_spin.setRange(0.0, 2.0)
+    temperature_spin.setSingleStep(0.05)
+    temperature_spin.setDecimals(2)
+    temperature_spin.setValue(config.llm.temperature)
+    temperature_spin.setToolTip("Sampling temperature. 0.0 = deterministic, 1.0 = creative")
+    form.addRow("Temperature:", temperature_spin)
+
+    num_ctx_spin = QtWidgets.QSpinBox()
+    num_ctx_spin.setRange(512, 131072)
+    num_ctx_spin.setSingleStep(1024)
+    num_ctx_spin.setValue(config.llm.num_ctx)
+    num_ctx_spin.setToolTip("Context window in tokens. Raise for many MCP tools or long conversations.")
+    form.addRow("Context window (tokens):", num_ctx_spin)
+
+    timeout_spin = QtWidgets.QSpinBox()
+    timeout_spin.setRange(10, 3600)
+    timeout_spin.setSuffix(" s")
+    timeout_spin.setValue(config.llm.request_timeout_seconds)
+    timeout_spin.setToolTip("Per-request timeout in seconds")
+    form.addRow("Request timeout:", timeout_spin)
+
+    return box, {
+        "url":         url_edit,
+        "model":       model_edit,
+        "temperature": temperature_spin,
+        "num_ctx":     num_ctx_spin,
+        "timeout":     timeout_spin,
+    }
+
+
+def _build_stt_section(config: Config) -> tuple:
+    """Speech-to-text enable + Whisper model-size picker."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Speech-to-text  (microphone input)")
+    layout = QtWidgets.QVBoxLayout(box)
+
+    enable_cb = QtWidgets.QCheckBox(
+        "Enable speech-to-text  (Push-to-Talk and wake word)")
+    enable_cb.setChecked(bool(config.stt.enabled))
+    layout.addWidget(enable_cb)
+
+    hint_lbl = QtWidgets.QLabel(
+        "When disabled, Javis still works via the text input on the Conversation tab. "
+        f"Cache: {_whisper_cache_dir(config)}"
+    )
+    hint_lbl.setObjectName("Hint")
+    hint_lbl.setWordWrap(True)
+    layout.addWidget(hint_lbl)
+
+    model_row = QtWidgets.QHBoxLayout()
+    model_row.addWidget(QtWidgets.QLabel("Model:"))
+    model_combo = QtWidgets.QComboBox()
+    cached_sizes = _whisper_cached_sizes(config)
+    for size_id, label, disk in _WHISPER_SIZES:
+        suffix = "  ✓ cached" if size_id in cached_sizes else f"  ({disk}, will download)"
+        model_combo.addItem(label + suffix, size_id)
+    for i in range(model_combo.count()):
+        if model_combo.itemData(i) == config.stt.model_size:
+            model_combo.setCurrentIndex(i)
+            break
+    model_row.addWidget(model_combo, 1)
+    layout.addLayout(model_row)
+
+    def _sync_model_picker() -> None:
+        model_combo.setEnabled(enable_cb.isChecked())
+
+    enable_cb.toggled.connect(lambda _: _sync_model_picker())
+    _sync_model_picker()
+
+    return box, {"enable_cb": enable_cb, "model_combo": model_combo}
+
+
+def _build_persona_section(config: Config, parent_dialog) -> tuple:
+    """Assistant persona / system-prompt picker."""
+    from PySide6 import QtWidgets
+
+    # Mutable single-item dict so the nested editor closure can update the value.
+    inline_prompt: dict = {"value": config.agent.system_prompt_inline or None}
+
+    box = QtWidgets.QGroupBox("Assistant persona  (system prompt)")
+    layout = QtWidgets.QVBoxLayout(box)
+
+    preset_row = QtWidgets.QHBoxLayout()
+    preset_row.addWidget(QtWidgets.QLabel("Preset:"))
+    preset_combo = QtWidgets.QComboBox()
+    for ps in agent_prompts.PRESETS:
+        preset_combo.addItem(ps.label, ps.key)
+    for i in range(preset_combo.count()):
+        if preset_combo.itemData(i) == (config.agent.preset or "default"):
+            preset_combo.setCurrentIndex(i)
+            break
+    preset_row.addWidget(preset_combo, 1)
+    custom_btn = QtWidgets.QPushButton("Custom…")
+    custom_btn.setToolTip("Edit the system prompt verbatim — overrides the preset")
+    preset_row.addWidget(custom_btn)
+    layout.addLayout(preset_row)
+
+    preset_desc_lbl = QtWidgets.QLabel("")
+    preset_desc_lbl.setObjectName("Hint")
+    preset_desc_lbl.setWordWrap(True)
+    layout.addWidget(preset_desc_lbl)
+
+    custom_active_lbl = QtWidgets.QLabel("")
+    custom_active_lbl.setObjectName("Hint")
+    custom_active_lbl.setWordWrap(True)
+    layout.addWidget(custom_active_lbl)
+
+    def _refresh_persona_labels() -> None:
+        ps = agent_prompts.get_preset(preset_combo.currentData())
+        preset_desc_lbl.setText(ps.description if ps else "")
+        custom_active_lbl.setText(
+            "Using custom prompt — overrides the preset above. "
+            "Click Custom… to edit or clear."
+            if inline_prompt["value"] else ""
+        )
+
+    preset_combo.currentIndexChanged.connect(lambda _: _refresh_persona_labels())
+
+    def _open_prompt_editor() -> None:
+        ps = agent_prompts.get_preset(preset_combo.currentData())
+        seed_text = inline_prompt["value"] or (ps.text if ps else agent_prompts.DEFAULT_PROMPT)
+
+        editor_dlg = QtWidgets.QDialog(parent_dialog)
+        editor_dlg.setWindowTitle("Custom system prompt")
+        editor_dlg.resize(720, 520)
+        editor_layout = QtWidgets.QVBoxLayout(editor_dlg)
+        editor_layout.addWidget(QtWidgets.QLabel(
+            "Edit the system prompt. Leave empty to clear and use the selected preset."))
+
+        text_edit = QtWidgets.QPlainTextEdit()
+        text_edit.setPlainText(seed_text)
+        pal = palette_for(config.gui.theme)
+        text_edit.setStyleSheet(
+            f"QPlainTextEdit {{ background:{pal['code_bg']}; color:{pal['text']}; "
+            f"border:1px solid {pal['border']}; border-radius:4px; padding:6px; "
+            "font-family: Consolas, monospace; font-size:10pt; }")
+        editor_layout.addWidget(text_edit, 1)
+
+        button_row = QtWidgets.QHBoxLayout()
+        clear_btn  = QtWidgets.QPushButton("Clear (use preset)")
+        cancel_btn = QtWidgets.QPushButton("Cancel")
+        save_btn   = QtWidgets.QPushButton("Save")
+        save_btn.setObjectName("Primary")
+        save_btn.setDefault(True)
+        button_row.addWidget(clear_btn)
+        button_row.addStretch(1)
+        button_row.addWidget(cancel_btn)
+        button_row.addWidget(save_btn)
+        editor_layout.addLayout(button_row)
+
+        def _on_save() -> None:
+            txt = text_edit.toPlainText().strip()
+            if txt and len(txt) < 10:
+                QtWidgets.QMessageBox.warning(
+                    editor_dlg, "Prompt too short",
+                    "The custom system prompt must be at least 10 characters, "
+                    "or clear it to use the preset.")
+                return
+            inline_prompt["value"] = txt or None
+            editor_dlg.accept()
+
+        save_btn.clicked.connect(_on_save)
+        cancel_btn.clicked.connect(editor_dlg.reject)
+        clear_btn.clicked.connect(lambda: (inline_prompt.update({"value": None}), editor_dlg.accept()))
+        if editor_dlg.exec():
+            _refresh_persona_labels()
+
+    custom_btn.clicked.connect(_open_prompt_editor)
+    _refresh_persona_labels()
+
+    return box, {"preset_combo": preset_combo, "inline_prompt": inline_prompt}
+
+
+def _build_audio_section(config: Config) -> tuple:
+    """Audio device selection and microphone gain."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Audio Devices")
+    form = QtWidgets.QFormLayout(box)
+    form.setVerticalSpacing(8)
+    form.setContentsMargins(10, 16, 10, 10)
+
+    # Parallel lists: display label (combo item text) and device index (None = system default).
+    device_labels:  list[str] = ["(system default)"]
+    device_indices: list      = [None]
+    try:
+        import sounddevice as sd
+        for i, dev in enumerate(sd.query_devices()):
+            device_labels.append(f"[{i}] {dev['name']}")
+            device_indices.append(i)
+    except Exception:
+        pass
+
+    input_combo  = QtWidgets.QComboBox()
+    output_combo = QtWidgets.QComboBox()
+    for label in device_labels:
+        input_combo.addItem(label)
+        output_combo.addItem(label)
+
+    for j, idx in enumerate(device_indices):
+        if idx == config.audio.input_device:
+            input_combo.setCurrentIndex(j)
+            break
+    for j, idx in enumerate(device_indices):
+        if idx == config.audio.output_device:
+            output_combo.setCurrentIndex(j)
+            break
+
+    form.addRow("Input device (mic):",       input_combo)
+    form.addRow("Output device (speakers):", output_combo)
+
+    gain_spin = QtWidgets.QDoubleSpinBox()
+    gain_spin.setRange(0.1, 10.0)
+    gain_spin.setSingleStep(0.1)
+    gain_spin.setDecimals(1)
+    gain_spin.setValue(config.audio.input_gain)
+    gain_spin.setToolTip("Microphone gain multiplier. 1.0 = no change.")
+    form.addRow("Input gain:", gain_spin)
+
+    # ── live mic level — catches "wrong microphone selected" before startup ──
+    level_bar = QtWidgets.QProgressBar()
+    level_bar.setRange(0, 100)
+    level_bar.setValue(0)
+    level_bar.setTextVisible(False)
+    level_bar.setFixedHeight(10)
+    level_bar.setToolTip("Live input level of the selected microphone — speak to test it")
+    level_lbl = QtWidgets.QLabel("")
+    level_lbl.setObjectName("Hint")
+    level_row = QtWidgets.QVBoxLayout()
+    level_row.setSpacing(2)
+    level_row.addWidget(level_bar)
+    level_row.addWidget(level_lbl)
+    form.addRow("Mic level:", level_row)
+
+    meter = _MicMeter(level_bar, level_lbl, gain_spin)
+
+    def _restart_meter() -> None:
+        meter.start(device_indices[input_combo.currentIndex()])
+
+    input_combo.currentIndexChanged.connect(lambda _i: _restart_meter())
+    _restart_meter()
+
+    return box, {
+        "input_combo":    input_combo,
+        "output_combo":   output_combo,
+        "device_indices": device_indices,
+        "input_gain":     gain_spin,
+        "mic_meter":      meter,
+    }
+
+
+class _MicMeter:
+    """Streams the selected input device and drives a level bar (~10 Hz)."""
+
+    def __init__(self, bar, label, gain_spin) -> None:
+        from PySide6 import QtCore
+        self._bar, self._label, self._gain = bar, label, gain_spin
+        self._stream = None
+        self._peak = 0.0
+        self._timer = QtCore.QTimer()
+        self._timer.setInterval(100)
+        self._timer.timeout.connect(self._tick)
+
+    def start(self, device) -> None:
+        self.stop()
+        try:
+            import numpy as np
+            import sounddevice as sd
+
+            def _cb(indata, _frames, _time, _status) -> None:
+                # Runs on the PortAudio thread — only touch the float.
+                self._peak = float(np.sqrt(np.mean(np.square(indata), dtype=np.float64)))
+
+            self._stream = sd.InputStream(
+                device=device, channels=1, samplerate=16000, dtype="float32",
+                blocksize=1600, callback=_cb,
+            )
+            self._stream.start()
+            self._label.setText("")
+            self._timer.start()
+        except Exception as exc:  # noqa: BLE001
+            self._stream = None
+            self._bar.setValue(0)
+            self._label.setText(f"Microphone unavailable: {exc}")
+
+    def stop(self) -> None:
+        self._timer.stop()
+        if self._stream is not None:
+            try:
+                self._stream.stop()
+                self._stream.close()
+            except Exception:  # noqa: BLE001
+                pass
+            self._stream = None
+        self._bar.setValue(0)
+
+    def _tick(self) -> None:
+        # RMS of speech at normal distance is ~0.02–0.2; scale so that
+        # reaches the upper half of the bar after the configured gain.
+        level = min(100, int(self._peak * float(self._gain.value()) * 400))
+        self._bar.setValue(level)
+
+
+def _build_wake_section(config: Config) -> tuple:
+    """Wake-word detection settings."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Wake Word")
+    form = QtWidgets.QFormLayout(box)
+    form.setVerticalSpacing(8)
+    form.setContentsMargins(10, 16, 10, 10)
+
+    enable_cb = QtWidgets.QCheckBox("Enable wake-word detection")
+    enable_cb.setChecked(bool(config.wake_word.enabled))
+    form.addRow("", enable_cb)
+
+    model_edit = QtWidgets.QLineEdit(config.wake_word.model or "hey_jarvis")
+    model_edit.setPlaceholderText("hey_jarvis")
+    model_edit.setToolTip("Wake word model name (from openwakeword)")
+    form.addRow("Model:", model_edit)
+
+    threshold_spin = QtWidgets.QDoubleSpinBox()
+    threshold_spin.setRange(0.0, 1.0)
+    threshold_spin.setSingleStep(0.05)
+    threshold_spin.setDecimals(2)
+    threshold_spin.setValue(config.wake_word.threshold)
+    threshold_spin.setToolTip("Detection threshold (0–1). Lower = more sensitive.")
+    form.addRow("Threshold:", threshold_spin)
+
+    vad_threshold_spin = QtWidgets.QDoubleSpinBox()
+    vad_threshold_spin.setRange(0.0, 1.0)
+    vad_threshold_spin.setSingleStep(0.05)
+    vad_threshold_spin.setDecimals(2)
+    vad_threshold_spin.setValue(config.wake_word.vad_threshold)
+    vad_threshold_spin.setToolTip("Voice-activity detection threshold for end-of-turn detection")
+    form.addRow("VAD threshold:", vad_threshold_spin)
+
+    silence_ms_spin = QtWidgets.QSpinBox()
+    silence_ms_spin.setRange(100, 5000)
+    silence_ms_spin.setSuffix(" ms")
+    silence_ms_spin.setValue(config.wake_word.silence_ms)
+    silence_ms_spin.setToolTip("Milliseconds of silence that ends the current turn")
+    form.addRow("Silence cutoff:", silence_ms_spin)
+
+    def _sync_wake_controls() -> None:
+        for w in (model_edit, threshold_spin, vad_threshold_spin, silence_ms_spin):
+            w.setEnabled(enable_cb.isChecked())
+
+    enable_cb.toggled.connect(lambda _: _sync_wake_controls())
+    _sync_wake_controls()
+
+    return box, {
+        "enable_cb":       enable_cb,
+        "model":           model_edit,
+        "threshold":       threshold_spin,
+        "vad_threshold":   vad_threshold_spin,
+        "silence_ms":      silence_ms_spin,
+    }
+
+
+def _build_speaker_id_section(config: Config) -> tuple:
+    """Speaker identification settings."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Speaker Identification")
+    form = QtWidgets.QFormLayout(box)
+    form.setVerticalSpacing(8)
+    form.setContentsMargins(10, 16, 10, 10)
+
+    enable_cb = QtWidgets.QCheckBox(
+        "Enable speaker identification  (requires speechbrain ≈250 MB)")
+    enable_cb.setChecked(bool(config.speaker_id.enabled))
+    form.addRow("", enable_cb)
+
+    threshold_spin = QtWidgets.QDoubleSpinBox()
+    threshold_spin.setRange(0.0, 1.0)
+    threshold_spin.setSingleStep(0.05)
+    threshold_spin.setDecimals(2)
+    threshold_spin.setValue(config.speaker_id.threshold)
+    threshold_spin.setToolTip(
+        "Cosine-similarity threshold for a positive speaker match (0–1). "
+        "Lower = more permissive (more false positives).")
+    form.addRow("Match threshold:", threshold_spin)
+
+    min_clips_spin = QtWidgets.QSpinBox()
+    min_clips_spin.setRange(1, 20)
+    min_clips_spin.setValue(config.speaker_id.min_enroll_clips)
+    min_clips_spin.setToolTip("Minimum audio clips required to create a speaker profile")
+    form.addRow("Min enroll clips:", min_clips_spin)
+
+    auto_enroll_cb = QtWidgets.QCheckBox("Auto-enroll unknown speakers")
+    auto_enroll_cb.setChecked(bool(config.speaker_id.auto_enroll))
+    auto_enroll_cb.setToolTip(
+        "When enabled, Javis asks new speakers for their name and creates a profile "
+        "from their first utterance.")
+    form.addRow("", auto_enroll_cb)
+
+    def _sync_speaker_controls() -> None:
+        for w in (threshold_spin, min_clips_spin, auto_enroll_cb):
+            w.setEnabled(enable_cb.isChecked())
+
+    enable_cb.toggled.connect(lambda _: _sync_speaker_controls())
+    _sync_speaker_controls()
+
+    return box, {
+        "enable_cb":        enable_cb,
+        "threshold":        threshold_spin,
+        "min_enroll_clips": min_clips_spin,
+        "auto_enroll":      auto_enroll_cb,
+    }
+
+
+def _build_memory_section(config: Config) -> tuple:
+    """Background memory-extraction settings."""
+    from PySide6 import QtWidgets
+
+    box = QtWidgets.QGroupBox("Memory Extraction")
+    form = QtWidgets.QFormLayout(box)
+    form.setVerticalSpacing(8)
+    form.setContentsMargins(10, 16, 10, 10)
+
+    hint_lbl = QtWidgets.QLabel(
+        "Memory extraction silently distills facts from conversations. "
+        "Leave 'Override model' empty to reuse the main LLM.")
+    hint_lbl.setObjectName("Hint")
+    hint_lbl.setWordWrap(True)
+    form.addRow(hint_lbl)
+
+    model_edit = QtWidgets.QLineEdit(config.memory.model or "")
+    model_edit.setPlaceholderText("empty = use main LLM")
+    model_edit.setToolTip("Optional dedicated extraction model, e.g. qwen2.5:14b")
+    form.addRow("Override model:", model_edit)
+
+    temperature_spin = QtWidgets.QDoubleSpinBox()
+    temperature_spin.setRange(0.0, 1.0)
+    temperature_spin.setSingleStep(0.05)
+    temperature_spin.setDecimals(2)
+    temperature_spin.setValue(config.memory.temperature)
+    temperature_spin.setToolTip("Low temperature (0.1) recommended for structured extraction")
+    form.addRow("Temperature:", temperature_spin)
+
+    num_ctx_spin = QtWidgets.QSpinBox()
+    num_ctx_spin.setRange(512, 32768)
+    num_ctx_spin.setSingleStep(512)
+    num_ctx_spin.setValue(config.memory.num_ctx)
+    form.addRow("Context window (tokens):", num_ctx_spin)
+
+    timeout_spin = QtWidgets.QSpinBox()
+    timeout_spin.setRange(10, 600)
+    timeout_spin.setSuffix(" s")
+    timeout_spin.setValue(
+        config.memory.timeout_seconds if config.memory.timeout_seconds is not None else 120)
+    form.addRow("Request timeout:", timeout_spin)
+
+    return box, {
+        "model":       model_edit,
+        "temperature": temperature_spin,
+        "num_ctx":     num_ctx_spin,
+        "timeout":     timeout_spin,
+    }
+
+
+def _build_voices_section(parent_dialog, parent_layout, config: Config) -> None:
+    """Voice-profile cleanup table (appends a group box to *parent_layout*)."""
+    from PySide6 import QtCore, QtWidgets
+
+    voices_dir = config.paths.voices_dir
+    profiles   = _scan_voice_profiles(voices_dir)
+
+    box = QtWidgets.QGroupBox(f"Voice profiles ({len(profiles)})")
+    layout = QtWidgets.QVBoxLayout(box)
+    layout.addWidget(QtWidgets.QLabel(
+        f"Voice profiles you (or audiobook ingest) created. "
+        f"Delete unused or corrupt ones to save disk.
"
+        f"{voices_dir}"
+    ))
+
+    table = QtWidgets.QTableWidget(len(profiles), 5)
+    table.setHorizontalHeaderLabels(["Name", "Refs", "Training clips", "Size", "Status"])
+    table.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
+    table.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
+    table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
+    table.verticalHeader().setVisible(False)
+    table.horizontalHeader().setStretchLastSection(True)
+
+    def _populate_row(row_idx: int, profile: dict) -> None:
+        name_item = QtWidgets.QTableWidgetItem(profile["name"])
+        name_item.setData(QtCore.Qt.UserRole, str(profile["path"]))
+        name_item.setToolTip(str(profile["path"]))
+        if profile["corrupt"]:
+            name_item.setForeground(QtCore.Qt.red)
+        table.setItem(row_idx, 0, name_item)
+        table.setItem(row_idx, 1, QtWidgets.QTableWidgetItem(str(profile["ref_count"])))
+        table.setItem(row_idx, 2, QtWidgets.QTableWidgetItem(str(profile["train_count"])))
+        table.setItem(row_idx, 3, QtWidgets.QTableWidgetItem(_format_bytes(profile["size_bytes"])))
+        status_text = ("corrupt: " + ", ".join(profile["reasons"])) if profile["corrupt"] else "ok"
+        status_item = QtWidgets.QTableWidgetItem(status_text)
+        if profile["corrupt"]:
+            status_item.setForeground(QtCore.Qt.red)
+        table.setItem(row_idx, 4, status_item)
+
+    for i, p in enumerate(profiles):
+        _populate_row(i, p)
+    table.resizeColumnsToContents()
+    layout.addWidget(table)
+
+    action_row = QtWidgets.QHBoxLayout()
+    btn_delete_selected = QtWidgets.QPushButton("Delete selected")
+    btn_delete_corrupt  = QtWidgets.QPushButton("Delete all corrupt")
+    btn_open_folder     = QtWidgets.QPushButton("Open folder")
+    btn_refresh         = QtWidgets.QPushButton("Refresh")
+    action_row.addWidget(btn_delete_selected)
+    action_row.addWidget(btn_delete_corrupt)
+    action_row.addStretch(1)
+    action_row.addWidget(btn_open_folder)
+    action_row.addWidget(btn_refresh)
+    layout.addLayout(action_row)
+    parent_layout.addWidget(box)
+
+    def _selected_profile_paths() -> list[Path]:
+        rows = sorted({item.row() for item in table.selectedIndexes()})
+        return [
+            Path(table.item(r, 0).data(QtCore.Qt.UserRole))
+            for r in rows if table.item(r, 0)
+        ]
+
+    def _delete_profile_paths(paths: list[Path]) -> None:
+        if not paths:
+            return
+        resp = QtWidgets.QMessageBox.question(
+            parent_dialog, "Delete voice profiles",
+            "Permanently delete the following profile directories?\n\n"
+            + "\n".join(f"• {p}" for p in paths),
+            QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
+        )
+        if resp != QtWidgets.QMessageBox.Yes:
+            return
+        errors: list[str] = []
+        for p in paths:
+            try:
+                shutil.rmtree(p)
+            except OSError as e:
+                errors.append(f"{p.name}: {e}")
+        if errors:
+            QtWidgets.QMessageBox.warning(
+                parent_dialog, "Some deletes failed", "\n".join(errors))
+        _refresh_profiles()
+
+    def _delete_corrupt_profiles() -> None:
+        corrupt_paths = [p["path"] for p in _scan_voice_profiles(voices_dir) if p["corrupt"]]
+        if not corrupt_paths:
+            QtWidgets.QMessageBox.information(
+                parent_dialog, "Nothing to do", "No corrupt voice profiles found.")
+            return
+        _delete_profile_paths(corrupt_paths)
+
+    def _refresh_profiles() -> None:
+        updated = _scan_voice_profiles(voices_dir)
+        table.setRowCount(len(updated))
+        for i, p in enumerate(updated):
+            _populate_row(i, p)
+        table.resizeColumnsToContents()
+        box.setTitle(f"Voice profiles ({len(updated)})")
+
+    def _open_voices_folder() -> None:
+        voices_dir.mkdir(parents=True, exist_ok=True)
+        QtCore.QProcess.startDetached("explorer.exe", [str(voices_dir)])
+
+    btn_delete_selected.clicked.connect(lambda: _delete_profile_paths(_selected_profile_paths()))
+    btn_delete_corrupt.clicked.connect(_delete_corrupt_profiles)
+    btn_open_folder.clicked.connect(_open_voices_folder)
+    btn_refresh.clicked.connect(_refresh_profiles)
+
+
+# ---------------------------------------------------------------------------
+# Main dialog
+# ---------------------------------------------------------------------------
+
+def show_startup_dialog(config: Config) -> Optional[Config]:
+    """Display the comprehensive configuration dialog.
+
+    Returns the (possibly mutated) config, or ``None`` if the user cancels.
+    """
+    from PySide6 import QtCore, QtWidgets
+
+    QtWidgets.QApplication.instance() or QtWidgets.QApplication([])
+
+    dlg = QtWidgets.QDialog()
+    dlg.setWindowTitle("Javis — Configuration")
+    dlg.resize(860, 820)
+    dlg.setStyleSheet(_dialog_qss(palette_for(config.gui.theme)))
+
+    root_layout = QtWidgets.QVBoxLayout(dlg)
+    root_layout.setContentsMargins(0, 0, 0, 0)
+    root_layout.setSpacing(0)
+
+    # ── header ─────────────────────────────────────────────────────────────
+    header = QtWidgets.QFrame()
+    header.setObjectName("Header")
+    header_layout = QtWidgets.QVBoxLayout(header)
+    header_layout.setContentsMargins(20, 14, 20, 14)
+    title_lbl    = QtWidgets.QLabel("Javis — Configuration")
+    title_lbl.setObjectName("Title")
+    subtitle_lbl = QtWidgets.QLabel(
+        "Review and edit all settings before starting. "
+        "Click ⚙ Save to config.yaml to persist changes, then Start to launch.")
+    subtitle_lbl.setObjectName("Subtitle")
+    subtitle_lbl.setWordWrap(True)
+    header_layout.addWidget(title_lbl)
+    header_layout.addWidget(subtitle_lbl)
+    root_layout.addWidget(header)
+
+    # ── scrollable body ─────────────────────────────────────────────────────
+    scroll = QtWidgets.QScrollArea()
+    scroll.setWidgetResizable(True)
+    scroll.setFrameShape(QtWidgets.QFrame.NoFrame)
+    scroll_content = QtWidgets.QWidget()
+    scroll.setWidget(scroll_content)
+    root_layout.addWidget(scroll, 1)
+
+    body_layout = QtWidgets.QVBoxLayout(scroll_content)
+    body_layout.setContentsMargins(20, 16, 20, 16)
+    body_layout.setSpacing(14)
+
+    # ── config sections ─────────────────────────────────────────────────────
+    llm_box,     llm_widgets     = _build_llm_section(config)
+    stt_box,     stt_widgets     = _build_stt_section(config)
+    persona_box, persona_widgets = _build_persona_section(config, dlg)
+    body_layout.addWidget(llm_box)
+    body_layout.addWidget(stt_box)
+    body_layout.addWidget(persona_box)
+
+    # ── TTS section ─────────────────────────────────────────────────────────
+    tts_group  = QtWidgets.QGroupBox("Text-to-speech  (Piper VITS)")
+    tts_layout = QtWidgets.QVBoxLayout(tts_group)
+
+    tts_enable_cb = QtWidgets.QCheckBox(
+        "Enable TTS  —  requires a trained piper_model.onnx in the voice profile directory")
+    tts_enable_cb.setChecked(bool(config.tts.enabled))
+    tts_layout.addWidget(tts_enable_cb)
+
+    tts_hint = QtWidgets.QLabel(
+        "Use the Voices tab to ingest audiobook material and train a Piper model.
"
+        "Voice training requires piper-train and espeak-ng on PATH.
"
+        "Runtime inference uses piper-tts (CPU, ~20 ms/sentence)."
+    )
+    tts_hint.setObjectName("Hint")
+    tts_hint.setWordWrap(True)
+    tts_layout.addWidget(tts_hint)
+
+    body_layout.addWidget(tts_group)
+
+    tts_widgets = {"enable_cb": tts_enable_cb}
+
+    # ── remaining config sections ───────────────────────────────────────────
+    audio_box,   audio_widgets   = _build_audio_section(config)
+    wake_box,    wake_widgets    = _build_wake_section(config)
+    speaker_box, speaker_widgets = _build_speaker_id_section(config)
+    memory_box,  memory_widgets  = _build_memory_section(config)
+    body_layout.addWidget(audio_box)
+    body_layout.addWidget(wake_box)
+    body_layout.addWidget(speaker_box)
+    body_layout.addWidget(memory_box)
+
+    # ── voice profiles cleanup ──────────────────────────────────────────────
+    _build_voices_section(dlg, body_layout, config)
+    body_layout.addStretch(1)
+
+    # ── footer ──────────────────────────────────────────────────────────────
+    footer = QtWidgets.QFrame()
+    footer.setObjectName("Footer")
+    footer_layout = QtWidgets.QHBoxLayout(footer)
+    footer_layout.setContentsMargins(20, 12, 20, 12)
+    footer_layout.setSpacing(8)
+
+    save_btn        = QtWidgets.QPushButton("⚙  Save to config.yaml")
+    save_status_lbl = QtWidgets.QLabel("")
+    save_status_lbl.setStyleSheet(f"color:{palette_for(config.gui.theme)['success']}; font-size:9pt;")
+    show_again_cb = QtWidgets.QCheckBox("Show this dialog on every start")
+    show_again_cb.setChecked(bool(config.gui.show_startup_dialog))
+    show_again_cb.setToolTip(
+        "Uncheck to launch straight into the main window next time.\n"
+        "Reopen any time via Tools → Startup configuration…")
+    cancel_btn = QtWidgets.QPushButton("Cancel")
+    start_btn  = QtWidgets.QPushButton("Start")
+    start_btn.setObjectName("Primary")
+    start_btn.setDefault(True)
+
+    footer_layout.addWidget(save_btn)
+    footer_layout.addWidget(save_status_lbl)
+    footer_layout.addStretch(1)
+    footer_layout.addWidget(show_again_cb)
+    footer_layout.addSpacing(12)
+    footer_layout.addWidget(cancel_btn)
+    footer_layout.addWidget(start_btn)
+    root_layout.addWidget(footer)
+
+    # ── helpers ─────────────────────────────────────────────────────────────
+
+    def _commit_widgets_to_config() -> None:
+        """Read every widget value and write it into the in-memory config."""
+        config.llm.base_url                = llm_widgets["url"].text().strip() or "http://localhost:11434"
+        config.llm.model                   = llm_widgets["model"].text().strip() or config.llm.model
+        config.llm.temperature             = llm_widgets["temperature"].value()
+        config.llm.num_ctx                 = llm_widgets["num_ctx"].value()
+        config.llm.request_timeout_seconds = llm_widgets["timeout"].value()
+
+        config.stt.enabled    = bool(stt_widgets["enable_cb"].isChecked())
+        config.stt.model_size = stt_widgets["model_combo"].currentData() or config.stt.model_size
+
+        config.agent.preset               = persona_widgets["preset_combo"].currentData() or config.agent.preset
+        config.agent.system_prompt_inline = persona_widgets["inline_prompt"]["value"]
+
+        config.tts.enabled = bool(tts_widgets["enable_cb"].isChecked())
+
+        config.audio.input_device  = audio_widgets["device_indices"][audio_widgets["input_combo"].currentIndex()]
+        config.audio.output_device = audio_widgets["device_indices"][audio_widgets["output_combo"].currentIndex()]
+        config.audio.input_gain    = audio_widgets["input_gain"].value()
+
+        config.wake_word.enabled       = bool(wake_widgets["enable_cb"].isChecked())
+        config.wake_word.model         = wake_widgets["model"].text().strip() or config.wake_word.model
+        config.wake_word.threshold     = wake_widgets["threshold"].value()
+        config.wake_word.vad_threshold = wake_widgets["vad_threshold"].value()
+        config.wake_word.silence_ms    = wake_widgets["silence_ms"].value()
+
+        config.speaker_id.enabled          = bool(speaker_widgets["enable_cb"].isChecked())
+        config.speaker_id.threshold        = speaker_widgets["threshold"].value()
+        config.speaker_id.min_enroll_clips = speaker_widgets["min_enroll_clips"].value()
+        config.speaker_id.auto_enroll      = bool(speaker_widgets["auto_enroll"].isChecked())
+
+        config.memory.model           = memory_widgets["model"].text().strip() or None
+        config.memory.temperature     = memory_widgets["temperature"].value()
+        config.memory.num_ctx         = memory_widgets["num_ctx"].value()
+        config.memory.timeout_seconds = memory_widgets["timeout"].value()
+
+        config.gui.show_startup_dialog = bool(show_again_cb.isChecked())
+
+    def _try_save() -> bool:
+        try:
+            _save_config_to_yaml(config)
+            return True
+        except Exception as exc:  # noqa: BLE001
+            QtWidgets.QMessageBox.warning(
+                dlg, "Save failed",
+                f"Settings were applied for this session but could not be written to "
+                f"config.yaml:\n\n{exc}")
+            return False
+
+    def _on_save() -> None:
+        _commit_widgets_to_config()
+        if _try_save():
+            save_status_lbl.setText("✓ Saved")
+            QtCore.QTimer.singleShot(3000, lambda: save_status_lbl.setText(""))
+
+    def _on_start() -> None:
+        _commit_widgets_to_config()
+        # Persist the "don't show again" choice without forcing a full save.
+        if not config.gui.show_startup_dialog and config.source_path is not None:
+            try:
+                update_config_file(config.source_path,
+                                   {"gui": {"show_startup_dialog": False}})
+            except Exception as exc:  # noqa: BLE001
+                QtWidgets.QMessageBox.warning(
+                    dlg, "Could not update config.yaml",
+                    f"The dialog will show again next start:\n\n{exc}")
+        dlg.accept()
+
+    save_btn.clicked.connect(_on_save)
+    start_btn.clicked.connect(_on_start)
+    cancel_btn.clicked.connect(dlg.reject)
+
+    try:
+        accepted = dlg.exec() == QtWidgets.QDialog.Accepted
+    finally:
+        audio_widgets["mic_meter"].stop()
+    return config if accepted else None
diff --git a/src/javis/gui/theme.py b/src/javis/gui/theme.py
new file mode 100644
index 0000000..dee371a
--- /dev/null
+++ b/src/javis/gui/theme.py
@@ -0,0 +1,42 @@
+"""Shared colour palettes for the main window and the startup dialog."""
+from __future__ import annotations
+
+# VS Code Dark Modern palette — orange accent.
+PAL_DARK = {
+    "bg": "#1e1e1e", "surface": "#252526", "surface2": "#2d2d2d",
+    "input": "#3c3c3c", "accent": "#DA7756", "accent2": "#e0886a",
+    "text": "#cccccc", "muted": "#858585", "border": "#3c3c3c",
+    "border2": "#454545", "purple": "#c586c0", "sel": "#3d2b1f",
+    "success": "#4ec994", "success_bg": "#1e2e1e", "success_border": "#3a5a3a",
+    "warn": "#dcdcaa", "warn_fg": "#e0a04a",
+    "error": "#e06060", "error_bg": "#3a1c1c", "error_border": "#5a2a2a",
+    "btn_hover": "#2a2d2e", "btn_pressed": "#37373d", "btn_disabled": "#9a9a9a",
+    "item_hover": "#2a2d2e", "handle": "#424242", "handle_hover": "#555555",
+    "code_bg": "#13111e", "code_fg": "#c0b0e8", "tool_bg": "#1e1c28",
+    "tool_fg": "#c0b0e8", "tool_muted": "#6a6280", "hr": "#333333",
+    "link": "#555550", "quote": "#aaaaaa", "user_name": "#b8b8b0",
+    "state_idle": "#3c3c3c", "tick": "#fff",
+}
+
+# VS Code Light palette.
+PAL_LIGHT = {
+    "bg": "#ffffff", "surface": "#f3f3f3", "surface2": "#ececec",
+    "input": "#ffffff", "accent": "#d1603c", "accent2": "#b84a28",
+    "text": "#1f1f1f", "muted": "#6e6e6e", "border": "#d0d0d0",
+    "border2": "#b8b8b8", "purple": "#a626a4", "sel": "#e8c9b8",
+    "success": "#2ea16a", "success_bg": "#e6f4ea", "success_border": "#a9d9b8",
+    "warn": "#c9a10a", "warn_fg": "#b8730a",
+    "error": "#d13f3f", "error_bg": "#fbe9e9", "error_border": "#e8b4b4",
+    "btn_hover": "#e6e6e6", "btn_pressed": "#d5d5d5", "btn_disabled": "#b0b0b0",
+    "item_hover": "#e6e6e6", "handle": "#c4c4c4", "handle_hover": "#a8a8a8",
+    "code_bg": "#f1ecfa", "code_fg": "#7c4fbd", "tool_bg": "#f4f0fa",
+    "tool_fg": "#7c4fbd", "tool_muted": "#8a80a8", "hr": "#dddddd",
+    "link": "#6e6e6e", "quote": "#555555", "user_name": "#4a4a4a",
+    "state_idle": "#8a8a8a", "tick": "#1f1f1f",
+}
+
+THEMES = {"dark": PAL_DARK, "light": PAL_LIGHT}
+
+
+def palette_for(name: str | None) -> dict:
+    return THEMES.get(name or "dark", PAL_DARK)
diff --git a/src/javis/llm/__init__.py b/src/javis/llm/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/javis/llm/conversation.py b/src/javis/llm/conversation.py
new file mode 100644
index 0000000..aa5e421
--- /dev/null
+++ b/src/javis/llm/conversation.py
@@ -0,0 +1,139 @@
+"""Conversation history with sliding-window truncation."""
+from __future__ import annotations
+
+from collections import deque
+from typing import Optional
+
+
+# Rough chars→tokens factor (~4 chars/token for English). Used only to bound
+# context size so Ollama doesn't keep re-processing a huge prompt every turn.
+_CHARS_PER_TOKEN = 4
+# Soft cap on the total characters of trimmed history. Generous default that
+# fits most local-LLM context windows (8k+ tokens).
+_DEFAULT_MAX_CHARS = 24000
+
+
+# Lazily-initialised tiktoken encoder shared across all Conversation instances.
+# ``False`` means "tried and unavailable" so we don't retry the import per call.
+_ENCODER: object | None = None
+
+
+def _get_encoder():
+    global _ENCODER
+    if _ENCODER is None:
+        try:
+            import tiktoken
+            _ENCODER = tiktoken.get_encoding("cl100k_base")
+        except Exception:
+            _ENCODER = False
+    return _ENCODER or None
+
+
+def count_tokens(text: str) -> int:
+    """Return the token count for *text*.
+
+    Uses ``tiktoken`` (cl100k_base) when installed for an accurate count that
+    holds up for non-English text and tool-call JSON; falls back to the
+    ``_CHARS_PER_TOKEN`` heuristic otherwise.
+    """
+    if not text:
+        return 0
+    enc = _get_encoder()
+    if enc is not None:
+        try:
+            return len(enc.encode(text))
+        except Exception:
+            pass
+    return len(text) // _CHARS_PER_TOKEN
+
+
+
+class Conversation:
+    def __init__(self, system_prompt: str, max_turns: int = 20,
+                 max_chars: int = _DEFAULT_MAX_CHARS):
+        self.system_prompt = system_prompt
+        self.max_turns = max_turns
+        self.max_chars = max_chars
+        self._history: deque[dict] = deque()
+
+    def add_user(self, text: str) -> None:
+        self._history.append({"role": "user", "content": text})
+        self._trim()
+
+    def add_assistant(self, text: str, tool_calls: Optional[list] = None) -> None:
+        msg: dict = {"role": "assistant", "content": text}
+        if tool_calls:
+            msg["tool_calls"] = tool_calls
+        self._history.append(msg)
+        self._trim()
+
+    def add_tool_result(self, tool_name: str, content: str, call_id: Optional[str] = None) -> None:
+        msg: dict = {"role": "tool", "content": content, "name": tool_name}
+        if call_id:
+            msg["tool_call_id"] = call_id
+        self._history.append(msg)
+        self._trim()
+
+    @staticmethod
+    def _msg_chars(m: dict) -> int:
+        n = len(m.get("content") or "")
+        for tc in m.get("tool_calls") or ():
+            args = tc.get("function", {}).get("arguments")
+            if args:
+                n += len(str(args))
+        return n
+
+    def _drop_oldest_turn(self) -> int:
+        """Drop the oldest complete turn from history (user → assistant → tool results).
+
+        Always drops at a ``user`` message boundary so we never orphan an
+        ``assistant`` tool-call message without its paired ``tool`` result(s),
+        which would produce an invalid message sequence and confuse the LLM.
+
+        Returns the total characters removed.
+        """
+        if not self._history:
+            return 0
+        chars = 0
+        # Drop the leading message (expected to be a user message).
+        chars += self._msg_chars(self._history.popleft())
+        # Continue dropping assistant / tool messages until we hit the next user
+        # message (or exhaust history).  This keeps tool-call + tool-result pairs
+        # together and prevents an orphaned ``role: tool`` message.
+        while self._history and self._history[0].get("role") != "user":
+            chars += self._msg_chars(self._history.popleft())
+        return chars
+
+    def _trim(self) -> None:
+        # Cap rough message count. Turns with tool calls have more messages than a plain
+        # user+assistant pair (user + assistant_tc + N tool results + assistant_final), so
+        # use a larger multiplier to avoid trimming aggressively on tool-heavy conversations.
+        while len(self._history) > self.max_turns * 8:
+            self._drop_oldest_turn()
+        # And cap by approximate prompt size so we don't blow the context window.
+        total = sum(self._msg_chars(m) for m in self._history) + len(self.system_prompt)
+        while total > self.max_chars and self._history:
+            total -= self._drop_oldest_turn()
+
+    @property
+    def approx_tokens(self) -> int:
+        # Accurate when tiktoken is installed; char-heuristic fallback otherwise.
+        total = count_tokens(self.system_prompt)
+        for m in self._history:
+            total += count_tokens(m.get("content") or "")
+            for tc in m.get("tool_calls") or ():
+                args = tc.get("function", {}).get("arguments")
+                if args:
+                    total += count_tokens(str(args))
+        return total
+
+    def messages(self) -> list[dict]:
+        return [{"role": "system", "content": self.system_prompt}] + list(self._history)
+
+    @property
+    def history(self) -> list[dict]:
+        """Read-only snapshot of the history messages (excludes the system prompt)."""
+        return list(self._history)
+
+    def clear(self) -> None:
+        self._history.clear()
diff --git a/src/javis/llm/hermes_client.py b/src/javis/llm/hermes_client.py
new file mode 100644
index 0000000..c53ad3a
--- /dev/null
+++ b/src/javis/llm/hermes_client.py
@@ -0,0 +1,147 @@
+"""OpenAI-compatible client for a remote Hermes Agent API server."""
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import Optional
+
+import httpx
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from .ollama_client import ChatResponse
+
+log = get_logger("llm.hermes")
+
+_RETRYABLE_CODES = {429, 503}
+_RETRY_DELAYS = (1.0, 2.0)
+
+
+class HermesClient:
+    """Call Hermes through its documented OpenAI-compatible API server."""
+
+    def __init__(self, config: Config):
+        memory = config.memory
+        if not memory.base_url:
+            raise ValueError("memory.base_url is required when memory.provider is 'hermes'")
+
+        self.base_url = memory.base_url.rstrip("/")
+        self.model = memory.model or "hermes-agent"
+        self.api_key_env = memory.api_key_env
+        self._client = httpx.AsyncClient(
+            timeout=float(memory.timeout_seconds or config.llm.request_timeout_seconds)
+        )
+
+    @property
+    def _chat_url(self) -> str:
+        if self.base_url.endswith("/v1"):
+            return f"{self.base_url}/chat/completions"
+        return f"{self.base_url}/v1/chat/completions"
+
+    def _headers(self) -> dict[str, str]:
+        api_key = os.environ.get(self.api_key_env, "").strip()
+        if not api_key:
+            raise RuntimeError(
+                f"Hermes API key is missing; set the {self.api_key_env} environment variable"
+            )
+        try:
+            api_key.encode("ascii")
+        except UnicodeEncodeError:
+            raise RuntimeError(
+                f"Hermes API key ({self.api_key_env}) contains non-ASCII characters "
+                "(e.g. umlauts) which HTTP headers do not allow. "
+                "Regenerate the TrueNAS Gateway API Key using only ASCII characters, "
+                "e.g. with: openssl rand -hex 32"
+            )
+        return {"Authorization": f"Bearer {api_key}"}
+
+    async def health(self) -> bool:
+        try:
+            root = self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url
+            response = await self._client.get(f"{root}/health", timeout=5.0)
+            return response.status_code == 200
+        except httpx.HTTPError:
+            return False
+
+    async def list_models(self) -> list[str]:
+        """Return authenticated model aliases advertised by Hermes."""
+        url = (
+            f"{self.base_url}/models"
+            if self.base_url.endswith("/v1")
+            else f"{self.base_url}/v1/models"
+        )
+        response = await self._client.get(url, headers=self._headers(), timeout=5.0)
+        response.raise_for_status()
+        return [
+            item["id"]
+            for item in response.json().get("data", [])
+            if isinstance(item, dict) and isinstance(item.get("id"), str)
+        ]
+
+    async def chat(
+        self,
+        messages: list[dict],
+        tools: Optional[list[dict]] = None,
+        stream: bool = False,
+        *,
+        model: Optional[str] = None,
+        temperature: Optional[float] = None,
+        num_ctx: Optional[int] = None,
+        timeout: Optional[float] = None,
+    ) -> ChatResponse:
+        if tools:
+            raise ValueError("Hermes memory extraction does not accept client-side tools")
+        if stream:
+            raise ValueError("Hermes memory extraction requires a non-streaming response")
+
+        payload = {
+            "model": model or self.model,
+            "messages": messages,
+            "stream": False,
+        }
+        if temperature is not None:
+            payload["temperature"] = temperature
+
+        client = self._client if timeout is None else httpx.AsyncClient(timeout=timeout)
+        try:
+            response = await self._post_with_retry(payload, client)
+        finally:
+            if client is not self._client:
+                await client.aclose()
+
+        if response.status_code >= 400:
+            log.error(f"Hermes {response.status_code}: {response.text[:500]}")
+            response.raise_for_status()
+
+        data = response.json()
+        choices = data.get("choices") or []
+        if not choices or not isinstance(choices[0], dict):
+            raise RuntimeError("Hermes response did not contain a completion choice")
+        message = choices[0].get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, str):
+            raise RuntimeError("Hermes response did not contain assistant text")
+        return ChatResponse(content=content, raw=data)
+
+    async def _post_with_retry(
+        self,
+        payload: dict,
+        client: httpx.AsyncClient,
+    ) -> httpx.Response:
+        for attempt in range(len(_RETRY_DELAYS) + 1):
+            try:
+                response = await client.post(
+                    self._chat_url,
+                    json=payload,
+                    headers=self._headers(),
+                )
+                if response.status_code not in _RETRYABLE_CODES or attempt >= len(_RETRY_DELAYS):
+                    return response
+            except (httpx.ConnectError, httpx.ReadTimeout):
+                if attempt >= len(_RETRY_DELAYS):
+                    raise
+            await asyncio.sleep(_RETRY_DELAYS[attempt])
+        raise RuntimeError("Hermes retry loop exited unexpectedly")
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
\ No newline at end of file
diff --git a/src/javis/llm/ollama_client.py b/src/javis/llm/ollama_client.py
new file mode 100644
index 0000000..d28a6e0
--- /dev/null
+++ b/src/javis/llm/ollama_client.py
@@ -0,0 +1,383 @@
+"""Async Ollama client with native tool calling."""
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Optional
+
+import httpx
+
+from ..core.config import Config
+from ..core.logger import get_logger
+
+log = get_logger("llm.ollama")
+
+# Transient HTTP status codes worth retrying (rate-limited or server overload).
+_RETRYABLE_CODES = {429, 503}
+# Maximum number of retry attempts for transient errors.
+_MAX_RETRIES = 2
+# Fixed backoff delays (seconds) per retry attempt.
+_RETRY_DELAYS = [1.0, 2.0]
+
+
+@dataclass
+class ToolCall:
+    name: str
+    arguments: dict[str, Any]
+    call_id: Optional[str] = None
+
+
+@dataclass
+class ChatResponse:
+    content: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    raw: dict = field(default_factory=dict)
+    # True when the request was silently retried WITHOUT tools (model rejected
+    # tool schemas or 500'd) — the caller must surface this degradation.
+    tools_dropped: bool = False
+
+
+def _looks_like_tools_rejection(body: str) -> bool:
+    b = body.lower()
+    return ("does not support tools" in b
+            or "tools not supported" in b
+            or ("tool" in b and ("not support" in b or "unsupported" in b)))
+
+
+def _looks_like_model_error(body: str) -> bool:
+    """Return True when a 500 body indicates a missing/unloadable model.
+
+    Ollama normally returns 404 for unknown models, but some versions or
+    quantisation paths return 500 with a descriptive message instead.  We
+    check for common phrases so we can surface a clear pull-and-retry hint
+    rather than silently stripping tools and masking the real problem.
+    """
+    b = body.lower()
+    return (
+        "no such file" in b
+        or "model not found" in b
+        or "pull model" in b
+        or "failed to load" in b
+        or "failed to init" in b
+        or "unable to load" in b
+        or ("model" in b and "not found" in b)
+    )
+
+
+def _looks_like_gpu_crash(body: str) -> bool:
+    """Return True when a 500 body indicates a GPU/runner crash.
+
+    These errors mean the llama runner process died (CUDA OOM, driver issue,
+    etc.).  Retrying the same request won't help — Ollama needs to be
+    restarted or a smaller / CPU-only model should be used instead.
+    """
+    b = body.lower()
+    return (
+        "runner process has terminated" in b
+        or "cuda error" in b
+        or "out of memory" in b
+        or "cudaerror" in b
+        or "gpu" in b and "error" in b
+        or "failed to allocate" in b
+        or "shared object initialization failed" in b
+    )
+
+
+class OllamaClient:
+    def __init__(self, config: Config):
+        self.config = config
+        self.base_url = config.llm.base_url.rstrip("/")
+        self._client = httpx.AsyncClient(timeout=config.llm.request_timeout_seconds)
+
+    async def aclose(self) -> None:
+        try:
+            await self._client.aclose()
+        except Exception as exc:
+            # Ignore cleanup errors (e.g., already closed, async generator issues)
+            log.debug(f"Error closing Ollama client: {exc}")
+
+    async def health(self) -> bool:
+        try:
+            r = await self._client.get(f"{self.base_url}/api/tags", timeout=5.0)
+            return r.status_code == 200
+        except Exception:
+            return False
+
+    async def list_models(self) -> list[str]:
+        """Return a list of available model names from Ollama."""
+        try:
+            r = await self._client.get(f"{self.base_url}/api/tags", timeout=5.0)
+            if r.status_code == 200:
+                data = r.json()
+                return [m["name"] for m in data.get("models", [])]
+        except Exception as e:
+            log.debug(f"Failed to list models: {e}")
+        return []
+
+    async def chat(
+        self,
+        messages: list[dict],
+        tools: Optional[list[dict]] = None,
+        stream: bool = False,
+        *,
+        model: Optional[str] = None,
+        temperature: Optional[float] = None,
+        num_ctx: Optional[int] = None,
+        timeout: Optional[float] = None,
+    ) -> ChatResponse:
+        """Non-streaming chat. For streaming use `chat_streaming`.
+
+        Keyword overrides ``model``, ``temperature``, ``num_ctx``, and
+        ``timeout`` take precedence over the values in ``config.llm`` when
+        provided.  Used by the memory-extraction subsystem to run a separate
+        (potentially smarter) model without changing the conversational model.
+        """
+        effective_model = model or self.config.llm.model
+        effective_num_ctx = num_ctx if num_ctx is not None else self.config.llm.num_ctx
+        # Temperature to use if tools get dropped mid-request (rejection/500
+        # fallback) and the call reverts to a plain conversational reply.
+        _no_tools_temp = temperature if temperature is not None else self.config.llm.temperature
+        if tools:
+            effective_temp = 0.0  # deterministic for tool calls
+        else:
+            effective_temp = _no_tools_temp
+
+        payload = {
+            "model": effective_model,
+            "messages": messages,
+            "stream": stream,
+            # Keep the model resident after this request so the next turn
+            # doesn't pay the full reload cost (Ollama default = 5 min).
+            "keep_alive": self.config.llm.keep_alive,
+            # Tool-call decisions are deterministic tasks — high temperature
+            # causes the model to narrate instead of emitting JSON tool calls.
+            "options": {
+                "temperature": effective_temp,
+                "num_ctx": effective_num_ctx,
+            },
+            # Qwen3 thinking mode.  False = skip chain-of-thought (faster for voice).
+            # Silently ignored by older models (qwen2.5, llama3, etc.).
+            "think": self.config.llm.thinking_mode,
+        }
+        if tools:
+            payload["tools"] = tools
+
+        client = (
+            httpx.AsyncClient(timeout=timeout)
+            if timeout is not None
+            else self._client
+        )
+        try:
+            r = await self._post_with_retry(f"{self.base_url}/api/chat", payload, client=client)
+        finally:
+            if timeout is not None:
+                try:
+                    await client.aclose()
+                except Exception as exc:
+                    # Ignore cleanup errors from one-shot clients
+                    log.debug(f"Error closing one-shot Ollama client: {exc}")
+
+        if r.status_code >= 400:
+            body = r.text[:500]
+            tools_dropped = False
+            # Some models (e.g. gemma3) don't support tool calling — retry without tools.
+            if tools and _looks_like_tools_rejection(body):
+                log.warning(
+                    f"model '{effective_model}' rejected tools; retrying without. "
+                    f"server said: {body}"
+                )
+                payload.pop("tools", None)
+                payload["options"]["temperature"] = _no_tools_temp
+                tools_dropped = True
+                r = await self._post_with_retry(f"{self.base_url}/api/chat", payload)
+            # 500 with runner crash usually means context overflow / OOM.
+            # BUT: if the body indicates a missing/unloadable model, raising
+            # immediately is more helpful than silently retrying without tools
+            # (which would also fail and mask the real problem).
+            elif r.status_code == 500 and tools:
+                body = r.text[:500]
+                if _looks_like_gpu_crash(body):
+                    log.error(f"Ollama 500 — GPU/runner crash for '{effective_model}': {body}")
+                    raise RuntimeError(
+                        f"Ollama GPU crash while running '{effective_model}'. "
+                        f"The llama runner process terminated (CUDA error or OOM).\n"
+                        f"Try: restart Ollama, use a smaller model, or add "
+                        f"'OLLAMA_GPU_LAYERS=0' to run on CPU.\n"
+                        f"Details: {body}"
+                    )
+                if _looks_like_model_error(body):
+                    log.error(f"Ollama 500 — model error for '{effective_model}': {body}")
+                    raise RuntimeError(
+                        f"Ollama model '{effective_model}' failed to load or is not installed.\n"
+                        f"Run: ollama pull {effective_model}\n"
+                        f"Details: {body}"
+                    )
+                log.warning(
+                    f"Ollama 500 (likely context overflow with {len(tools)} tools, "
+                    f"body={body!r}); retrying without tools."
+                )
+                payload.pop("tools", None)
+                payload["options"]["temperature"] = _no_tools_temp
+                tools_dropped = True
+                r = await self._post_with_retry(f"{self.base_url}/api/chat", payload)
+            if r.status_code >= 400:
+                log.error(f"Ollama {r.status_code}: {r.text[:500]}")
+                if r.status_code == 404:
+                    raise RuntimeError(
+                        f"Ollama model '{effective_model}' not found. "
+                        f"Run: ollama pull {effective_model}"
+                    )
+                r.raise_for_status()
+        else:
+            tools_dropped = False
+
+        data = r.json()
+        msg = data.get("message", {})
+        resp = _parse_message(msg, data)
+        resp.tools_dropped = tools_dropped
+        return resp
+
+    async def _post_with_retry(
+        self,
+        url: str,
+        payload: dict,
+        client: Optional["httpx.AsyncClient"] = None,
+    ) -> httpx.Response:
+        """POST *payload* to *url*, retrying on transient errors.
+
+        Retries on HTTP 429/503 (rate-limit / overload) and connection errors.
+        Other errors are raised immediately.  Pass an explicit *client* to use
+        a one-shot client with a custom timeout (e.g. for memory extraction).
+        """
+        _client = client or self._client
+        last_exc: Optional[Exception] = None
+        for attempt in range(_MAX_RETRIES + 1):
+            try:
+                r = await _client.post(url, json=payload)
+                if r.status_code in _RETRYABLE_CODES and attempt < _MAX_RETRIES:
+                    delay = _RETRY_DELAYS[attempt]
+                    log.warning(
+                        f"Ollama returned {r.status_code} "
+                        f"({'rate limited' if r.status_code == 429 else 'server overload'}); "
+                        f"retrying in {delay:.0f}s (attempt {attempt + 1}/{_MAX_RETRIES})"
+                    )
+                    await asyncio.sleep(delay)
+                    continue
+                return r
+            except httpx.ConnectError as exc:
+                last_exc = exc
+                if attempt < _MAX_RETRIES:
+                    delay = _RETRY_DELAYS[attempt]
+                    log.warning(
+                        f"Ollama connection failed: {exc}; "
+                        f"retrying in {delay:.0f}s (attempt {attempt + 1}/{_MAX_RETRIES})"
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    raise
+        # Should only be reached if _MAX_RETRIES == 0 and a ConnectError fires.
+        raise last_exc  # type: ignore[misc]
+
+    async def chat_streaming(
+        self,
+        messages: list[dict],
+        tools: Optional[list[dict]] = None,
+        on_delta: Optional[Callable[[str], Awaitable[None]]] = None,
+    ) -> ChatResponse:
+        """Streamed chat that accumulates content + tool_calls and reports
+        content deltas via ``on_delta``. Falls back to non-streaming + no-tools
+        if the model rejects tool calling.
+        """
+        payload = {
+            "model": self.config.llm.model,
+            "messages": messages,
+            "stream": True,
+            # num_ctx must match the non-streaming path — omitting it makes
+            # Ollama fall back to its small default context, silently
+            # truncating long histories on conversational (no-tools) turns.
+            "options": {
+                "temperature": self.config.llm.temperature,
+                "num_ctx": self.config.llm.num_ctx,
+            },
+            # Keep the model loaded for 1 h after this request (see chat()).
+            "keep_alive": self.config.llm.keep_alive,
+            # Qwen3 thinking mode.  False = skip chain-of-thought (faster for voice).
+            # Silently ignored by older models (qwen2.5, llama3, etc.).
+            "think": self.config.llm.thinking_mode,
+        }
+        if tools:
+            payload["tools"] = tools
+
+        content_parts: list[str] = []
+        tool_calls: list[ToolCall] = []
+        last_obj: dict = {}
+        try:
+            async with self._client.stream(
+                "POST", f"{self.base_url}/api/chat", json=payload,
+            ) as r:
+                if r.status_code >= 400:
+                    body = (await r.aread()).decode("utf-8", "replace")[:500]
+                    if tools and _looks_like_tools_rejection(body):
+                        log.warning(
+                            f"model '{self.config.llm.model}' rejected tools "
+                            f"during streaming; retrying without tools (non-streaming)."
+                        )
+                        return await self.chat(messages, tools=None, stream=False)
+                    log.error(f"Ollama {r.status_code}: {body}")
+                    r.raise_for_status()
+                try:
+                    async for line in r.aiter_lines():
+                        if not line.strip():
+                            continue
+                        try:
+                            obj = json.loads(line)
+                        except json.JSONDecodeError:
+                            log.warning(f"bad chunk: {line[:80]}")
+                            continue
+                        last_obj = obj
+                        msg = obj.get("message", {}) or {}
+                        delta = msg.get("content", "") or ""
+                        if delta:
+                            content_parts.append(delta)
+                            if on_delta is not None:
+                                try:
+                                    await on_delta(delta)
+                                except Exception:  # noqa: BLE001
+                                    log.exception("on_delta handler failed")
+                        for tc in msg.get("tool_calls", []) or []:
+                            tool_calls.append(_to_tool_call(tc))
+                        if obj.get("done"):
+                            break
+                except GeneratorExit:
+                    # Stream was cancelled or closed — this is expected on context exit
+                    pass
+        except httpx.HTTPStatusError:
+            raise
+        except Exception as exc:
+            # Suppress or log non-critical stream cleanup errors
+            # but re-raise actual streaming failures (not GeneratorExit-related)
+            if not isinstance(exc, GeneratorExit):
+                log.debug(f"Stream exception during streaming: {exc}")
+        return ChatResponse(
+            content="".join(content_parts),
+            tool_calls=tool_calls,
+            raw=last_obj,
+        )
+
+
+def _to_tool_call(obj: dict) -> ToolCall:
+    fn = obj.get("function", {})
+    name = fn.get("name", "")
+    args = fn.get("arguments", {}) or {}
+    if isinstance(args, str):
+        try:
+            args = json.loads(args)
+        except json.JSONDecodeError:
+            args = {"_raw": args}
+    return ToolCall(name=name, arguments=args, call_id=obj.get("id"))
+
+
+def _parse_message(msg: dict, raw: dict) -> ChatResponse:
+    tool_calls = [_to_tool_call(tc) for tc in (msg.get("tool_calls") or [])]
+    return ChatResponse(content=msg.get("content", ""), tool_calls=tool_calls, raw=raw)
diff --git a/src/javis/mcp_client/__init__.py b/src/javis/mcp_client/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/javis/mcp_client/host.py b/src/javis/mcp_client/host.py
new file mode 100644
index 0000000..9cf9811
--- /dev/null
+++ b/src/javis/mcp_client/host.py
@@ -0,0 +1,463 @@
+"""MCP host: spawns configured servers and aggregates their tools."""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shutil
+import sys
+import tempfile
+from contextlib import AsyncExitStack
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from .schema_bridge import mcp_tool_to_ollama, parse_qualified_name, validate_tool_name
+# Import mcp at module level — must happen before the Qt/PySide6 event loop
+# starts, otherwise PySide6's shibokensupport intercepts pydantic attribute
+# access during lazy import and triggers a circular-import crash in
+# pydantic._internal._validators.
+from mcp import ClientSession, StdioServerParameters, types as mcp_types
+from mcp.client.stdio import stdio_client
+
+log = get_logger("mcp")
+
+
+_PREREQ_HINTS: dict[str, str] = {
+    "npx": (
+        "Node.js is not installed (npx not found on PATH). "
+        "Install Node.js LTS from https://nodejs.org/ (or `winget install OpenJS.NodeJS.LTS`) "
+        "and restart Javis."
+    ),
+    "npm": (
+        "Node.js is not installed (npm not found on PATH). "
+        "Install Node.js LTS from https://nodejs.org/ (or `winget install OpenJS.NodeJS.LTS`)."
+    ),
+    "uvx": (
+        "uv is not installed (uvx not found on PATH). "
+        "Install with `pip install uv` or `winget install astral-sh.uv`, then restart Javis."
+    ),
+    "uv": (
+        "uv is not installed. "
+        "Install with `pip install uv` or `winget install astral-sh.uv`."
+    ),
+    "node": (
+        "Node.js is not installed. "
+        "Install from https://nodejs.org/ (or `winget install OpenJS.NodeJS.LTS`)."
+    ),
+    "python": "Python is not on PATH. Install Python 3.10+ and ensure it is on PATH.",
+    "pwsh": "PowerShell 7 (pwsh) is not on PATH. Install from https://aka.ms/powershell.",
+}
+
+
+def _command_exists(command: str) -> Optional[str]:
+    """Return the resolved path of *command* or None if not found.
+
+    On Windows, also probes common shim extensions (.cmd/.bat/.exe) because
+    ``shutil.which`` may miss them depending on PATHEXT.
+    """
+    located = shutil.which(command)
+    if located:
+        return located
+    if sys.platform == "win32":
+        for ext in (".cmd", ".bat", ".exe", ".ps1"):
+            located = shutil.which(command + ext)
+            if located:
+                return located
+    # Also accept an absolute or relative path that points directly at a file.
+    try:
+        p = Path(command)
+        if p.is_file():
+            return str(p)
+    except OSError:
+        return None
+    return None
+
+
+def _missing_prereq_message(command: str) -> str:
+    key = Path(command).name.lower()
+    for ext in (".cmd", ".bat", ".exe", ".ps1"):
+        if key.endswith(ext):
+            key = key[: -len(ext)]
+            break
+    hint = _PREREQ_HINTS.get(key)
+    if hint:
+        return hint
+    return (
+        f"command '{command}' not found on PATH. Install it or update mcp_servers.json "
+        "to use an absolute path."
+    )
+
+
+def _resolve_windows_command(command: str, args: list[str]) -> tuple[str, list[str]]:
+    """On Windows, MCP SDK spawns without a shell, so .cmd/.bat shims like
+    ``npx``/``uvx`` are not found. Wrap them via ``cmd /c``.
+    """
+    if sys.platform != "win32":
+        return command, args
+    # If the command resolves to a .cmd/.bat (or has no extension and only a shim
+    # exists on PATH), use cmd /c.
+    located = shutil.which(command)
+    needs_shim = (
+        located is None
+        or located.lower().endswith((".cmd", ".bat"))
+        or command.lower() in {"npx", "uvx", "npm", "pnpm", "yarn"}
+    )
+    if needs_shim:
+        return "cmd", ["/c", command, *args]
+    return command, args
+
+
+@dataclass
+class ServerSpec:
+    name: str
+    command: str
+    args: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+    transport: str = "stdio"
+    approval: str = "ask"  # ask | allow | deny
+
+
+@dataclass
+class ConnectedServer:
+    spec: ServerSpec
+    session: Any
+    tools: list[Any] = field(default_factory=list)
+    # Each server owns its own exit stack so it can be torn down and rebuilt
+    # independently of the others (see MCPHost._reconnect).
+    stack: Optional[AsyncExitStack] = None
+
+
+def load_server_specs(path: Path, default_approval: str = "ask") -> list[ServerSpec]:
+    if not path.exists():
+        log.warning(f"MCP config not found: {path}")
+        return []
+    data = json.loads(path.read_text(encoding="utf-8"))
+    raw = data.get("servers") or data.get("mcpServers") or {}
+    specs: list[ServerSpec] = []
+    for name, conf in raw.items():
+        specs.append(ServerSpec(
+            name=name,
+            command=conf.get("command", ""),
+            args=list(conf.get("args", [])),
+            env=dict(conf.get("env", {})),
+            transport=conf.get("transport", "stdio"),
+            approval=conf.get("approval", default_approval),
+        ))
+    return specs
+
+
+ApprovalCallback = Any  # async (server, tool, args) -> bool
+
+
+class MCPHost:
+    """Connects to MCP servers, lists tools, dispatches calls."""
+
+    def __init__(self, config: Config):
+        self.config = config
+        self.servers: dict[str, ConnectedServer] = {}
+        # Serialises (re)connection so a burst of failed calls doesn't spawn
+        # several concurrent respawns of the same server.
+        self._conn_lock: asyncio.Lock = asyncio.Lock()
+        # Cached specs (including ones that failed initial connect) so a dead
+        # server can be respawned later on demand.
+        self._specs: dict[str, ServerSpec] = {}
+        # Per-tool decisions remembered for the session.
+        self._allow_cache: set[tuple[str, str]] = set()
+        # Denials only stick for the current turn (see clear_denials) so a
+        # single "Deny" click doesn't lock the tool out for the whole session.
+        self._deny_cache: set[tuple[str, str]] = set()
+        self.approval_cb: Optional[ApprovalCallback] = None
+        self._warned_no_approval_cb = False
+
+    def allow_tool(self, server: str, tool: str) -> None:
+        """Remember an 'always allow' decision for ``(server, tool)`` this session."""
+        self._allow_cache.add((server, tool))
+
+    def clear_allowed(self) -> None:
+        """Forget all session 'always allow' approvals."""
+        self._allow_cache.clear()
+
+    def clear_denials(self) -> None:
+        """Forget per-turn denials; the orchestrator calls this at each new turn."""
+        self._deny_cache.clear()
+
+    def clear_approvals(self) -> None:
+        """Forget all session approval decisions (both allow and deny)."""
+        self._allow_cache.clear()
+        self._deny_cache.clear()
+
+    async def start(self) -> None:
+        config_path = Path(self.config.mcp.servers_config)
+        if not config_path.is_absolute():
+            config_path = Path.cwd() / config_path
+        specs = load_server_specs(config_path, self.config.mcp.approval_default)
+        self._specs = {spec.name: spec for spec in specs}
+
+        for spec in specs:
+            conn = await self._connect_one(spec)
+            if conn is not None:
+                self.servers[spec.name] = conn
+
+    async def _connect_one(self, spec: "ServerSpec") -> Optional[ConnectedServer]:
+        """Spawn and initialise a single MCP server on its own exit stack.
+
+        Returns a :class:`ConnectedServer` on success, or ``None`` on any
+        failure (missing binary, unsupported transport, handshake error).
+        """
+        if not spec.command:
+            log.warning(f"server '{spec.name}' has no command, skipping")
+            return None
+        if spec.transport != "stdio":
+            log.warning(f"server '{spec.name}': unsupported transport '{spec.transport}'")
+            return None
+        # Pre-flight: ensure the command actually exists. Without this the
+        # MCP SDK only reports a cryptic "Connection closed" when the spawned
+        # process exits immediately because the binary is missing.
+        if _command_exists(spec.command) is None:
+            log.error(
+                f"MCP server '{spec.name}' disabled: {_missing_prereq_message(spec.command)}"
+            )
+            return None
+
+        stack = AsyncExitStack()
+        await stack.__aenter__()
+        stderr_file: Optional[Any] = None
+        stderr_path: Optional[Path] = None
+        try:
+            cmd, cmd_args = _resolve_windows_command(spec.command, spec.args)
+            env = {**os.environ, **(spec.env or {})}
+            params = StdioServerParameters(
+                command=cmd, args=cmd_args, env=env
+            )
+            # The MCP SDK passes ``errlog`` directly to ``subprocess.Popen(stderr=...)``,
+            # which requires a real OS file descriptor. ``io.StringIO`` raises
+            # ``UnsupportedOperation: fileno`` on Windows, so use a tempfile instead.
+            # delete=False because the file must stay open while the process runs;
+            # we clean it up manually in the finally block.
+            stderr_file = tempfile.NamedTemporaryFile(
+                mode="w+",
+                encoding="utf-8",
+                errors="replace",
+                delete=False,
+                prefix=f"mcp-{spec.name}-",
+                suffix=".stderr.log",
+            )
+            stderr_path = Path(stderr_file.name)
+            read, write = await stack.enter_async_context(
+                stdio_client(params, errlog=stderr_file)
+            )
+            session = await stack.enter_async_context(ClientSession(read, write))
+            await session.initialize()
+            # MCP list_tools is paginated — follow next_cursor until exhausted.
+            tools: list = []
+            cursor = None
+            while True:
+                lp = mcp_types.PaginatedRequestParams(cursor=cursor) if cursor else None
+                page = await session.list_tools(params=lp)
+                tools.extend(page.tools)
+                cursor = page.next_cursor
+                if not cursor:
+                    break
+            log.info(f"connected MCP server '{spec.name}' — {len(tools)} tools")
+            return ConnectedServer(spec=spec, session=session, tools=tools, stack=stack)
+        except Exception as e:
+            # Tear down the half-open stack so we don't leak the subprocess.
+            try:
+                await stack.aclose()
+            except Exception:
+                pass
+            stderr_tail = ""
+            if stderr_path is not None:
+                try:
+                    if stderr_file is not None:
+                        stderr_file.flush()
+                    stderr_tail = stderr_path.read_text(
+                        encoding="utf-8", errors="replace"
+                    ).strip()
+                except OSError:
+                    stderr_tail = ""
+            if stderr_tail:
+                tail_lines = stderr_tail.splitlines()[-8:]
+                stderr_tail = " | ".join(tail_lines)
+                log.error(
+                    f"failed to start MCP server '{spec.name}': {e} — stderr: {stderr_tail}"
+                )
+            else:
+                log.error(f"failed to start MCP server '{spec.name}': {e}")
+            return None
+        finally:
+            # Close and delete the temp stderr file regardless of outcome.
+            if stderr_file is not None:
+                try:
+                    stderr_file.close()
+                except OSError:
+                    pass
+            if stderr_path is not None:
+                try:
+                    stderr_path.unlink(missing_ok=True)
+                except OSError:
+                    pass
+
+    async def _reconnect(self, name: str, failed_session: Any) -> bool:
+        """Respawn a server whose session (``failed_session``) just errored.
+
+        Serialised via ``_conn_lock``.  Uses session identity to detect when a
+        concurrent caller already replaced the dead session, so repeated deaths
+        are each handled exactly once.  Returns True if a live session exists.
+        """
+        spec = self._specs.get(name)
+        if spec is None:
+            return False
+        async with self._conn_lock:
+            cur = self.servers.get(name)
+            # A concurrent waiter already reconnected this exact death.
+            if cur is not None and cur.session is not failed_session:
+                return True
+            if cur is not None and cur.stack is not None:
+                # The subprocess is presumed dead; a messy close (including
+                # anyio cross-task cancel-scope errors) is acceptable here.
+                try:
+                    await cur.stack.aclose()
+                except Exception as e:  # noqa: BLE001
+                    log.debug(f"[mcp] ignoring teardown error for dead '{name}': {e}")
+            log.info(f"[mcp] reconnecting server '{name}' …")
+            conn = await self._connect_one(spec)
+            if conn is None:
+                self.servers.pop(name, None)
+                log.warning(f"[mcp] reconnect of '{name}' failed")
+                return False
+            self.servers[name] = conn
+            return True
+
+    async def stop(self) -> None:
+        # Close each server's stack independently (best-effort).
+        for name, srv in list(self.servers.items()):
+            if srv.stack is not None:
+                try:
+                    await srv.stack.aclose()
+                except Exception as e:  # noqa: BLE001
+                    log.warning(f"error closing MCP server '{name}': {e}")
+        self.servers.clear()
+
+    def ollama_tools(self) -> list[dict]:
+        tools: list[dict] = []
+        for srv in self.servers.values():
+            for t in srv.tools:
+                tools.append(mcp_tool_to_ollama(srv.spec.name, t))
+        return tools
+
+    def status_snapshot(self) -> dict:
+        """Return a UI-friendly snapshot of connected MCP servers and their tools."""
+        servers = []
+        total_tools = 0
+        for name, srv in self.servers.items():
+            tool_names = [getattr(t, "name", "?") for t in srv.tools]
+            total_tools += len(tool_names)
+            servers.append({
+                "name": name,
+                "command": srv.spec.command,
+                "approval": srv.spec.approval,
+                "tools": tool_names,
+            })
+        return {"servers": servers, "tool_count": total_tools}
+
+    async def call(self, qualified_name: str, arguments: dict, *,
+                   pre_approved: bool = False) -> str:
+        """Dispatch a tool call.
+
+        *pre_approved* skips the per-call approval dialog for this invocation.
+        Use it when an upstream privilege/ACL check has already authorised the
+        tool (e.g. PrivilegeMiddleware in the LLM loop) so the user isn't
+        interrupted by a redundant confirmation dialog.
+        """
+        # Build available tools mapping for validation/recovery
+        available_tools = {
+            srv_name: [getattr(t, "name", "?") for t in srv.tools]
+            for srv_name, srv in self.servers.items()
+        }
+        
+        # Validate tool name with recovery fallback
+        server, tool = validate_tool_name(qualified_name, available_tools)
+        
+        if server not in self.servers:
+            return f"ERROR: unknown server '{server}' for tool '{tool}'"
+
+        if (server, tool) in self._deny_cache:
+            return f"ERROR: tool '{qualified_name}' denied by user"
+
+        # Strip None values — the LLM sometimes passes null for optional
+        # parameters, which fails strict JSON Schema validation in MCP servers.
+        arguments = {k: v for k, v in arguments.items() if v is not None}
+
+        srv = self.servers[server]
+        approval = srv.spec.approval
+        if approval == "deny":
+            return f"ERROR: tool '{qualified_name}' permanently denied in config"
+        if approval == "ask" and (server, tool) not in self._allow_cache:
+            if pre_approved:
+                # Upstream privilege check already authorised — skip dialog.
+                log.debug(f"[approval] pre-approved '{qualified_name}' (privilege gate passed)")
+            else:
+                if self.approval_cb is not None:
+                    try:
+                        allowed = await self.approval_cb(server, tool, arguments)
+                    except Exception as e:
+                        log.warning(f"approval callback error: {e}")
+                        allowed = False
+                else:
+                    # No UI to ask (headless): 'ask' must fail closed, otherwise
+                    # the configured policy silently degrades to 'allow'.
+                    allowed = False
+                    if not self._warned_no_approval_cb:
+                        self._warned_no_approval_cb = True
+                        log.warning(
+                            "MCP approval policy is 'ask' but no approval UI is available "
+                            "(headless) — such tool calls are denied. Set approval to "
+                            "'allow' per server in mcp_servers.json to permit them.")
+                if not allowed:
+                    self._deny_cache.add((server, tool))
+                    return f"ERROR: user denied tool call '{qualified_name}'"
+                self._allow_cache.add((server, tool))
+
+        try:
+            result = await asyncio.wait_for(
+                srv.session.call_tool(tool, arguments),
+                timeout=self.config.mcp.call_timeout_seconds,
+            )
+        except asyncio.TimeoutError:
+            return f"ERROR: tool '{qualified_name}' timed out"
+        except Exception as e:
+            # The session likely died (subprocess crashed / stream closed).
+            # Respawn the server once and retry so a single crash isn't fatal.
+            log.warning(f"[{qualified_name}] MCP call failed ({e}) — attempting reconnect")
+            if await self._reconnect(server, srv.session):
+                try:
+                    srv = self.servers[server]
+                    result = await asyncio.wait_for(
+                        srv.session.call_tool(tool, arguments),
+                        timeout=self.config.mcp.call_timeout_seconds,
+                    )
+                    log.info(f"[{qualified_name}] succeeded after reconnect")
+                    return _result_to_text(result)
+                except Exception as e2:
+                    log.exception(f"[{qualified_name}] MCP tool call failed after reconnect")
+                    return f"ERROR: tool '{qualified_name}' failed after reconnect: {e2}"
+            return f"ERROR: tool '{qualified_name}' failed: {e}"
+
+        return _result_to_text(result)
+
+
+def _result_to_text(result: Any) -> str:
+    parts: list[str] = []
+    for c in getattr(result, "content", []) or []:
+        text = getattr(c, "text", None)
+        if text:
+            parts.append(text)
+        else:
+            parts.append(str(c))
+    if getattr(result, "isError", False):
+        return "ERROR: " + "\n".join(parts)
+    return "\n".join(parts) if parts else ""
diff --git a/src/javis/mcp_client/schema_bridge.py b/src/javis/mcp_client/schema_bridge.py
new file mode 100644
index 0000000..4b34f4b
--- /dev/null
+++ b/src/javis/mcp_client/schema_bridge.py
@@ -0,0 +1,65 @@
+"""Translate MCP tool definitions ↔ Ollama tool format."""
+from __future__ import annotations
+
+from typing import Any
+
+
+def mcp_tool_to_ollama(server_name: str, tool: Any) -> dict:
+    """Convert an `mcp.types.Tool` to Ollama's `tools` entry.
+
+    Tool names are prefixed with `{server}__` so the dispatcher can route them.
+    """
+    schema = getattr(tool, "inputSchema", None) or {"type": "object", "properties": {}}
+    return {
+        "type": "function",
+        "function": {
+            "name": f"{server_name}__{tool.name}",
+            "description": (tool.description or "")[:1024],
+            "parameters": schema,
+        },
+    }
+
+
+def parse_qualified_name(name: str) -> tuple[str, str]:
+    """Split a `{server}__{tool}` name. Returns ('', name) if unprefixed."""
+    if "__" in name:
+        server, _, tool = name.partition("__")
+        return server, tool
+    return "", name
+
+
+def validate_tool_name(name: str, available_tools: dict[str, list[str]] | None = None) -> tuple[str, str]:
+    """Parse and validate a tool name, with fallback/recovery for malformed names.
+    
+    Args:
+        name: Tool name to validate (expected format: "server__tool_name")
+        available_tools: Optional dict mapping server names to lists of available tool names.
+                        Used to attempt recovery of malformed tool names.
+    
+    Returns:
+        Tuple of (server, tool) for valid names, or ('', name) if malformed and no recovery possible.
+    
+    Attempts recovery strategies:
+    1. If name is just a server name with no tool suffix, tries to find the first tool from that server
+    2. Logs warnings when recovery is applied
+    """
+    # First try normal parsing
+    server, tool = parse_qualified_name(name.strip())
+    
+    # If valid format found, return it
+    if server and tool:
+        return server, tool
+    
+    # Try recovery strategies if available_tools provided and name looks like it might be a server name
+    if available_tools and not "__" in name:
+        # The LLM might have generated just a server name without the tool suffix
+        # Try to find a matching server and use its first tool
+        name_lower = name.lower().replace('_', '-')  # normalize underscores to dashes
+        for server_name, tools in available_tools.items():
+            if server_name.lower() == name_lower or server_name.lower().replace('_', '-') == name_lower:
+                if tools:
+                    # Return the server with its first tool as a fallback
+                    return server_name, tools[0]
+    
+    # No recovery possible
+    return "", name
diff --git a/src/javis/stt/__init__.py b/src/javis/stt/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/javis/stt/transcriber.py b/src/javis/stt/transcriber.py
new file mode 100644
index 0000000..fe5b7de
--- /dev/null
+++ b/src/javis/stt/transcriber.py
@@ -0,0 +1,252 @@
+"""Speech-to-text using faster-whisper."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Optional
+
+import numpy as np
+
+from ..core.config import Config
+from ..core.logger import get_logger
+
+log = get_logger("stt")
+
+from ..core.hardware import add_nvidia_dll_dirs as _add_nvidia_dll_dirs
+
+# Register NVIDIA pip-package DLL dirs on PATH as early as possible so that
+# ctranslate2's internal LoadLibrary calls find cublas64_12.dll etc. before
+# the first CUDA operation is attempted.
+_add_nvidia_dll_dirs()
+
+
+def _patch_hf_symlinks() -> None:
+    """Fall back to shutil.copyfile when os.symlink raises OSError.
+
+    On Windows without Developer Mode the symlink privilege
+    (SeCreateSymbolicLinkPrivilege) is not granted to normal users, so
+    huggingface_hub's cache raises OSError [WinError 1314] while building
+    the snapshot directory.  Newer versions of huggingface_hub handle this
+    internally; this patch covers older versions that don't.
+    The patch is idempotent — applying it twice is harmless.
+    """
+    try:
+        import shutil
+        import huggingface_hub.file_download as _fd
+
+        _orig = _fd._create_symlink
+
+        def _safe_symlink(src: str, dst: str, new_blob: bool = False) -> None:
+            try:
+                _orig(src, dst, new_blob=new_blob)
+            except OSError:
+                import os
+                os.makedirs(os.path.dirname(dst), exist_ok=True)
+                try:
+                    os.remove(dst)
+                except OSError:
+                    pass
+                shutil.copyfile(src, dst)
+
+        if _fd._create_symlink is not _safe_symlink:
+            _fd._create_symlink = _safe_symlink
+    except Exception:
+        pass  # If patching fails, let the original code run as-is
+
+
+@dataclass
+class TranscriptionResult:
+    text: str
+    language: str
+    duration: float
+
+
+class Transcriber:
+    def __init__(self, config: Config):
+        self.config = config
+        self._model = None
+        self._batched = None
+        self._force_cpu: bool = False  # set True after a CUDA runtime DLL failure
+
+    def _resolve_device_and_compute(self) -> tuple[str, str]:
+        """Resolve the best device + compute_type for CTranslate2 / faster-whisper.
+
+        faster-whisper runs on CTranslate2, which has its own CUDA runtime
+        completely independent of PyTorch.  Using torch.cuda.is_available() to
+        gate CUDA is therefore wrong — a CPU-only PyTorch build would always
+        force int8 even when a GPU is present and CTranslate2 can use it.
+
+        This method queries CTranslate2 directly and falls back gracefully.
+        """
+        requested_device = self.config.stt.device   # "auto" | "cuda" | "cuda:N" | "cpu"
+        requested_compute = self.config.stt.compute_type
+        want_cuda = (
+            requested_device in ("auto", "cuda")
+            or requested_device.startswith("cuda:")
+        ) and not self._force_cpu
+
+        # Ensure nvidia-* pip package DLL dirs are on the Windows DLL search
+        # path before CTranslate2 tries to open cublas64_12.dll etc.
+        if want_cuda:
+            added = _add_nvidia_dll_dirs()
+            if added:
+                log.debug(f"STT: registered NVIDIA DLL dirs: {added}")
+
+        try:
+            import ctranslate2
+        except ImportError:
+            log.warning("ctranslate2 not importable — defaulting to cpu/int8")
+            return "cpu", "int8"
+
+        # --- GPU path -------------------------------------------------
+        if want_cuda:
+            try:
+                cuda_types = ctranslate2.get_supported_compute_types("cuda")
+                if cuda_types:
+                    # CTranslate2 can reach the GPU — pick the requested or
+                    # best available compute type.
+                    preferred = (
+                        requested_compute
+                        if requested_compute in cuda_types
+                        else "float16"
+                    )
+                    if preferred not in cuda_types:
+                        preferred = cuda_types[0]
+                    log.info(
+                        f"STT: CTranslate2 CUDA available — "
+                        f"supported={cuda_types}, using {preferred}"
+                    )
+                    return "cuda", preferred
+            except Exception as exc:
+                log.debug(f"STT: CUDA probe failed ({exc}) — falling back to CPU")
+
+        # --- CPU path -------------------------------------------------
+        # Preference order: int8_float32 > int8_float16 > int8
+        # int8_float32 uses int8 weights with float32 accumulation — faster
+        # than plain int8 on CPUs with AVX2 / AVX-512 (most modern x86).
+        try:
+            cpu_types = ctranslate2.get_supported_compute_types("cpu")
+        except Exception:
+            cpu_types = ["int8"]
+
+        for candidate in ("int8_float32", "int8_float16", "int8"):
+            if candidate in cpu_types:
+                log.info(f"STT: CPU mode, compute_type={candidate} "
+                         f"(supported={cpu_types})")
+                return "cpu", candidate
+
+        return "cpu", "int8"
+
+    def load(self) -> None:
+        if self._model is not None:
+            return
+        _patch_hf_symlinks()
+        from faster_whisper import WhisperModel
+        device, compute = self._resolve_device_and_compute()
+        log.info(f"loading faster-whisper: {self.config.stt.model_size} on {device}/{compute}")
+        if device == "cpu" and self.config.stt.model_size in (
+            "large", "large-v1", "large-v2", "large-v3", "large-v3-turbo"
+        ):
+            log.warning(
+                f"STT: using '{self.config.stt.model_size}' on CPU — expect very high latency "
+                "(30–120 s per utterance). Consider 'distil-large-v3.5', 'distil-large-v3', "
+                "'medium', or 'small' for CPU use, or enable GPU via 'stt.device: cuda' in config.yaml."
+            )
+        # Parse optional GPU index from "cuda:N" notation.
+        device_index = 0
+        raw_device = self.config.stt.device
+        if raw_device.startswith("cuda:"):
+            try:
+                device_index = int(raw_device.split(":", 1)[1])
+            except (ValueError, IndexError):
+                device_index = 0
+
+        self._model = WhisperModel(
+            self.config.stt.model_size,
+            device=device,
+            device_index=device_index,
+            compute_type=compute,
+            cpu_threads=0,  # 0 = use all available cores
+            download_root=str(self.config.paths.models_dir / "whisper"),
+        )
+        # BatchedInferencePipeline (faster-whisper >=1.1) runs transcription in
+        # parallel across the batch dimension — roughly 4x faster on GPU with
+        # no meaningful WER change.  CPU path skips this since batching adds
+        # overhead without a parallel compute backend.
+        self._batched: "object | None" = None
+        if device == "cuda" and self.config.stt.use_batched:
+            try:
+                from faster_whisper import BatchedInferencePipeline
+                self._batched = BatchedInferencePipeline(model=self._model)
+                log.info(
+                    f"faster-whisper BatchedInferencePipeline enabled "
+                    f"(batch_size={self.config.stt.batch_size})"
+                )
+            except Exception as exc:
+                log.warning(
+                    f"BatchedInferencePipeline unavailable ({exc}); "
+                    "falling back to sequential transcription"
+                )
+        log.info("faster-whisper loaded")
+
+    def _transcribe_sync(self, audio: np.ndarray, sample_rate: int) -> TranscriptionResult:
+        if self._model is None:
+            raise RuntimeError("Transcriber not loaded — call load() first")
+        if sample_rate != 16000:
+            # resample_poly is O(N) and ~100x faster than scipy.signal.resample (FFT-based).
+            from fractions import Fraction
+            import scipy.signal
+            frac = Fraction(16000, sample_rate).limit_denominator(1000)
+            audio = scipy.signal.resample_poly(audio, frac.numerator, frac.denominator).astype(np.float32)
+            sample_rate = 16000
+
+        # Common keyword arguments accepted by both the standard WhisperModel
+        # and BatchedInferencePipeline.transcribe().
+        transcribe_kwargs: dict = dict(
+            language=self.config.stt.language,
+            beam_size=self.config.stt.beam_size,
+            condition_on_previous_text=self.config.stt.condition_on_previous_text,
+            without_timestamps=True,
+        )
+
+        def _run_transcribe() -> tuple:
+            if self._batched is not None:
+                return self._batched.transcribe(
+                    audio,
+                    batch_size=self.config.stt.batch_size,
+                    **transcribe_kwargs,
+                )
+            return self._model.transcribe(
+                audio,
+                vad_filter=False,  # VAD already applied upstream
+                **transcribe_kwargs,
+            )
+
+        try:
+            segments, info = _run_transcribe()
+        except RuntimeError as exc:
+            _msg = str(exc)
+            if any(k in _msg.lower() for k in ("dll", "cublas", "cannot be loaded")):
+                log.warning(
+                    f"STT: CUDA runtime library missing ({exc}) — reloading model on CPU"
+                )
+                self._model = None
+                self._batched = None  # also discard the CUDA-bound batched pipeline
+                self._force_cpu = True
+                self.load()
+                segments, info = _run_transcribe()
+            else:
+                raise
+
+        text = "".join(s.text for s in segments).strip()
+        return TranscriptionResult(
+            text=text,
+            language=info.language,
+            duration=info.duration,
+        )
+
+    async def transcribe(self, audio: np.ndarray, sample_rate: Optional[int] = None) -> TranscriptionResult:
+        if self._model is None:
+            await asyncio.to_thread(self.load)
+        sr = sample_rate or self.config.audio.sample_rate
+        return await asyncio.to_thread(self._transcribe_sync, audio, sr)
diff --git a/src/javis/stt/voice_profiler.py b/src/javis/stt/voice_profiler.py
new file mode 100644
index 0000000..5c1232c
--- /dev/null
+++ b/src/javis/stt/voice_profiler.py
@@ -0,0 +1,413 @@
+"""Speaker identification via ECAPA-TDNN embeddings (speechbrain).
+
+Profiles are stored as L2-normalised numpy vectors in::
+
+    ~/.javis/voice_profiles/{safe_name}.npy
+    ~/.javis/voice_profiles/index.json
+
+Identification is cosine-similarity against all enrolled profiles.
+
+Falls back to ``NullProfiler`` silently when speechbrain / torch is not
+available so the rest of the agent continues to work unchanged.
+"""
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from fnmatch import fnmatch as _fnmatch
+from pathlib import Path
+from typing import Optional, Union
+
+import numpy as np
+
+from ..core.config import Config
+from ..core.logger import get_logger
+
+log = get_logger("voice_profiler")
+
+_INDEX_FILE = "index.json"
+_MODEL_SOURCE = "speechbrain/spkrec-ecapa-voxceleb"
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+def _cosine(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity between two vectors (handles un-normalised input)."""
+    na, nb = np.linalg.norm(a), np.linalg.norm(b)
+    if na == 0 or nb == 0:
+        return 0.0
+    return float(np.dot(a, b) / (na * nb))
+
+
+def _l2(v: np.ndarray) -> np.ndarray:
+    n = np.linalg.norm(v)
+    return v / n if n > 0 else v
+
+
+# Minimum utterance duration for a reliable embedding.  Shorter clips are
+# passed through but logged as a warning — ECAPA was trained on ≥ 1 s segments.
+_MIN_EMBED_DURATION_S = 0.5
+
+# Target RMS level applied before embedding.  Normalising to a fixed RMS makes
+# embeddings robust to microphone-gain changes and volume variation between
+# enrollment and identification sessions.
+_TARGET_RMS = 0.05
+
+
+def _normalize_rms(audio: np.ndarray) -> np.ndarray:
+    """Scale *audio* so its RMS equals ``_TARGET_RMS``.  Silence is returned as-is."""
+    rms = float(np.sqrt(np.mean(audio.astype(np.float64) ** 2)))
+    if rms < 1e-8:
+        return audio
+    return (audio * (_TARGET_RMS / rms)).astype(np.float32)
+
+
+# ── privileges ────────────────────────────────────────────────────────────────
+
+@dataclass
+class SpeakerPrivileges:
+    """Per-speaker access control for tools and responses.
+
+    Attributes:
+        can_respond:    If ``False`` the turn is silently dropped; Javis says nothing.
+        allowed_tools:  ``"*"`` grants all tools; a list of fnmatch patterns restricts
+                        access to only the matching tool names.
+        denied_tools:   Patterns that are *always* blocked regardless of ``allowed_tools``.
+                        Evaluated before ``allowed_tools`` (deny wins).
+    """
+
+    can_respond: bool = True
+    allowed_tools: Union[str, list[str]] = "*"
+    denied_tools: list[str] = field(default_factory=list)
+
+    def allows_tool(self, tool_name: str) -> bool:
+        """Return ``True`` if *tool_name* is permitted under these privileges."""
+        for pat in self.denied_tools:
+            if _fnmatch(tool_name, pat):
+                return False
+        if self.allowed_tools == "*":
+            return True
+        return any(_fnmatch(tool_name, pat) for pat in self.allowed_tools)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "SpeakerPrivileges":
+        return cls(
+            can_respond=bool(d.get("can_respond", True)),
+            allowed_tools=d.get("allowed_tools", "*"),
+            denied_tools=list(d.get("denied_tools", [])),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "can_respond": self.can_respond,
+            "allowed_tools": self.allowed_tools,
+            "denied_tools": self.denied_tools,
+        }
+
+
+# ── null fallback ─────────────────────────────────────────────────────────────
+
+class NullProfiler:
+    """Silent no-op used when speechbrain / torch is unavailable."""
+
+    def load(self) -> None:
+        pass
+
+    def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:  # noqa: ARG002
+        return np.zeros(192, dtype=np.float32)
+
+    def enroll(self, name: str, clips: list, sr: int) -> None:  # noqa: ARG002
+        log.warning("[profiler] NullProfiler — speechbrain unavailable, enroll skipped")
+
+    def identify(self, audio_np: np.ndarray, sr: int) -> Optional[tuple[str, float]]:  # noqa: ARG002
+        return None
+
+    def list_profiles(self) -> list[dict]:
+        return []
+
+    def delete_profile(self, name: str) -> bool:  # noqa: ARG002
+        return False
+
+    def get_privileges(self, name: str) -> SpeakerPrivileges:  # noqa: ARG002
+        return SpeakerPrivileges()
+
+    def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None:  # noqa: ARG002
+        pass
+
+
+# ── main class ───────────────────────────────────────────────────────────────
+
+class VoiceProfiler:
+    """Speaker identification using ECAPA-TDNN embeddings from speechbrain.
+
+    Usage::
+
+        profiler = VoiceProfiler(config)
+        profiler.load()                              # load model once
+        profiler.enroll("Alice", [clip1, clip2], sr) # register speaker
+        result = profiler.identify(audio, sr)        # (name, confidence) | None
+    """
+
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        self._profiles_dir: Path = config.paths.voice_profiles_dir
+        self._encoder = None          # speechbrain EncoderClassifier or None
+        self._profiles: dict[str, np.ndarray] = {}   # name → L2-normed embedding
+        self._index: list[dict] = []                  # metadata records
+
+    # ── lifecycle ─────────────────────────────────────────────────────────────
+
+    def load(self) -> None:
+        """Load the ECAPA-TDNN encoder and all enrolled profiles from disk."""
+        self._profiles_dir.mkdir(parents=True, exist_ok=True)
+        self._load_encoder()
+        self._load_index()
+        log.info(f"[profiler] ready — {len(self._profiles)} profile(s) loaded")
+
+    def _load_encoder(self) -> None:
+        try:
+            import os
+            import warnings
+
+            # Silence benign third-party warnings before any speechbrain /
+            # huggingface_hub imports happen.
+            #
+            # 1. HuggingFace symlinks — we handle this in _patch_hf_symlinks;
+            #    the env-var disables the cache-warning banner.
+            os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+            # 2. speechbrain.pretrained alias — we already import from
+            #    speechbrain.inference, but speechbrain's own internals still
+            #    trigger this on some versions.
+            warnings.filterwarnings(
+                "ignore",
+                message=r".*speechbrain\.pretrained.*deprecated.*",
+                category=UserWarning,
+            )
+            # 3. speechbrain Pretrainer symlinks — harmless on Windows because
+            #    model files are copied rather than symlinked.
+            warnings.filterwarnings(
+                "ignore",
+                message=r".*Requested Pretrainer collection using symlinks.*",
+                category=UserWarning,
+            )
+
+            from speechbrain.inference.classifiers import EncoderClassifier
+            import torch  # noqa: F401 — verify torch present
+
+            # SpeechBrain's run_opts parser expects "cuda:N", not bare "cuda".
+            device = "cuda:0" if self._try_cuda() else "cpu"
+            savedir = str(self.config.paths.models_dir / "ecapa-tdnn")
+            log.info(f"[profiler] loading ECAPA-TDNN on {device} → {savedir}")
+            self._encoder = EncoderClassifier.from_hparams(
+                source=_MODEL_SOURCE,
+                savedir=savedir,
+                run_opts={"device": device},
+            )
+            log.info("[profiler] ECAPA-TDNN loaded")
+        except Exception as exc:
+            log.warning(
+                f"[profiler] speechbrain unavailable ({exc}) — "
+                "speaker identification disabled"
+            )
+            self._encoder = None
+
+    @staticmethod
+    def _try_cuda() -> bool:
+        try:
+            import torch
+            return torch.cuda.is_available()
+        except Exception:
+            return False
+
+    # ── index I/O ─────────────────────────────────────────────────────────────
+
+    def _load_index(self) -> None:
+        idx = self._profiles_dir / _INDEX_FILE
+        if not idx.exists():
+            self._index = []
+            self._profiles = {}
+            return
+        try:
+            with open(idx, encoding="utf-8") as f:
+                self._index = json.load(f)
+        except Exception as exc:
+            log.warning(f"[profiler] index read failed: {exc}")
+            self._index = []
+
+        self._profiles = {}
+        for entry in self._index:
+            npy = self._profiles_dir / entry["file"]
+            if npy.exists():
+                try:
+                    self._profiles[entry["name"]] = np.load(str(npy))
+                except Exception as exc:
+                    log.warning(f"[profiler] could not load {npy.name}: {exc}")
+
+    def _save_index(self) -> None:
+        idx = self._profiles_dir / _INDEX_FILE
+        tmp = idx.with_suffix(".json.part")
+        try:
+            with open(tmp, "w", encoding="utf-8") as f:
+                json.dump(self._index, f, ensure_ascii=False, indent=2)
+            tmp.replace(idx)
+        except Exception as exc:
+            log.error(f"[profiler] index write failed: {exc}")
+
+    # ── embedding ─────────────────────────────────────────────────────────────
+
+    def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
+        """Return a L2-normalised ECAPA-TDNN speaker embedding for *audio_np*."""
+        if self._encoder is None:
+            return np.zeros(192, dtype=np.float32)
+        import torch
+
+        # Warn on very short clips — ECAPA embeddings degrade below ~0.5 s.
+        duration = len(audio_np) / sr
+        if duration < _MIN_EMBED_DURATION_S:
+            log.warning(
+                f"[profiler] embed: clip is only {duration:.2f}s "
+                f"(< {_MIN_EMBED_DURATION_S}s) — embedding may be unreliable"
+            )
+
+        # Resample to 16 kHz if needed (ECAPA was trained at 16 kHz).
+        if sr != 16000:
+            from fractions import Fraction
+            import scipy.signal
+            frac = Fraction(16000, sr).limit_denominator(1000)
+            audio_np = scipy.signal.resample_poly(
+                audio_np, frac.numerator, frac.denominator
+            ).astype(np.float32)
+
+        # Normalise amplitude so embeddings are consistent across mic gains.
+        audio_np = _normalize_rms(audio_np)
+
+        wav = torch.tensor(audio_np.astype(np.float32)).unsqueeze(0)  # [1, T]
+        wav_lens = torch.tensor([1.0])
+        with torch.no_grad():
+            emb = self._encoder.encode_batch(wav, wav_lens)  # [1, 1, D]
+        return _l2(emb.squeeze().cpu().numpy())
+
+    # ── CRUD ──────────────────────────────────────────────────────────────────
+
+    def enroll(self, name: str, clips: list[np.ndarray], sr: int) -> None:
+        """Enroll *name* by averaging embeddings extracted from *clips*."""
+        if self._encoder is None:
+            log.warning("[profiler] enroll skipped — encoder not loaded")
+            return
+        vecs = [self.embed(c, sr) for c in clips]
+        avg = _l2(np.mean(vecs, axis=0))
+
+        # Build a filesystem-safe filename from the profile name.
+        safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in name)
+        npy_path = self._profiles_dir / f"{safe}.npy"
+        np.save(str(npy_path), avg)
+        self._profiles[name] = avg
+
+        # Replace existing entry if the name already exists.
+        self._index = [e for e in self._index if e["name"] != name]
+        self._index.append({
+            "name": name,
+            "file": f"{safe}.npy",
+            "enrolled_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
+            "clip_count": len(clips),
+        })
+        self._save_index()
+        log.info(f"[profiler] enrolled '{name}' from {len(clips)} clip(s)")
+
+    def identify(
+        self, audio_np: np.ndarray, sr: int
+    ) -> Optional[tuple[str, float]]:
+        """Return *(name, confidence)* for the best-matching enrolled profile.
+
+        Returns ``None`` when no profiles are enrolled or the best score is
+        below ``config.speaker_id.threshold``.
+        """
+        if self._encoder is None or not self._profiles:
+            return None
+        query = self.embed(audio_np, sr)
+        threshold = self.config.speaker_id.threshold
+
+        scores: dict[str, float] = {
+            name: _cosine(query, emb) for name, emb in self._profiles.items()
+        }
+        best_name = max(scores, key=scores.__getitem__)
+        best_score = scores[best_name]
+
+        # Always log all candidate scores so identification failures are visible
+        # in normal (INFO-level) logs without needing to enable DEBUG mode.
+        score_str = ", ".join(
+            f"{n}={s:.3f}" for n, s in sorted(scores.items(), key=lambda x: -x[1])
+        )
+        if best_score >= threshold:
+            log.info(f"[profiler] identified '{best_name}' [{score_str}] threshold={threshold}")
+            return best_name, float(best_score)
+        log.info(
+            f"[profiler] unknown speaker [{score_str}] threshold={threshold} — no match"
+        )
+        return None
+
+    def list_profiles(self) -> list[dict]:
+        """Return a list of profile metadata dicts from the index."""
+        return list(self._index)
+
+    def refine(self, name: str, clip: np.ndarray, sr: int) -> None:
+        """Blend a new voice sample into *name*'s existing profile.
+
+        Uses a running-average update so that each additional clip shifts the
+        stored centroid by ``1 / (n + 1)`` — more clips = more stable profile.
+        The updated embedding is L2-normalised and written back to disk.
+        """
+        if self._encoder is None:
+            return
+        entry = next((e for e in self._index if e["name"] == name), None)
+        if entry is None or name not in self._profiles:
+            log.warning(f"[profiler] refine: no profile found for '{name}'")
+            return
+        new_emb = self.embed(clip, sr)
+        n = entry.get("clip_count", 1)
+        current = self._profiles[name]
+        # Running centroid: (current * n + new) / (n+1), then L2-normalise.
+        blended = _l2(current * n + new_emb)
+        npy_path = self._profiles_dir / entry["file"]
+        np.save(str(npy_path), blended)
+        self._profiles[name] = blended
+        entry["clip_count"] = n + 1
+        entry["last_seen"] = time.strftime("%Y-%m-%dT%H:%M:%S")
+        self._save_index()
+        log.debug(f"[profiler] refined '{name}' — centroid now over {n + 1} clip(s)")
+
+    def delete_profile(self, name: str) -> bool:
+        """Remove *name*'s embedding from disk and memory."""
+        entry = next((e for e in self._index if e["name"] == name), None)
+        if entry is None:
+            return False
+        npy = self._profiles_dir / entry["file"]
+        try:
+            npy.unlink(missing_ok=True)
+        except Exception as exc:
+            log.warning(f"[profiler] delete {npy.name} failed: {exc}")
+        self._index = [e for e in self._index if e["name"] != name]
+        self._profiles.pop(name, None)
+        self._save_index()
+        log.info(f"[profiler] deleted profile '{name}'")
+        return True
+
+    # ── privileges ────────────────────────────────────────────────────────────
+
+    def get_privileges(self, name: str) -> SpeakerPrivileges:
+        """Return the stored ``SpeakerPrivileges`` for *name*, or an unrestricted default."""
+        entry = next((e for e in self._index if e["name"] == name), None)
+        if entry is None:
+            return SpeakerPrivileges()
+        return SpeakerPrivileges.from_dict(entry.get("privileges", {}))
+
+    def set_privileges(self, name: str, privileges: SpeakerPrivileges) -> None:
+        """Persist updated *privileges* for *name* to ``index.json``."""
+        entry = next((e for e in self._index if e["name"] == name), None)
+        if entry is None:
+            log.warning(f"[profiler] set_privileges: no profile found for '{name}'")
+            return
+        entry["privileges"] = privileges.to_dict()
+        self._save_index()
+        log.info(f"[profiler] privileges updated for '{name}'")
diff --git a/src/javis/tts/__init__.py b/src/javis/tts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/javis/tts/base.py b/src/javis/tts/base.py
new file mode 100644
index 0000000..d07f4f7
--- /dev/null
+++ b/src/javis/tts/base.py
@@ -0,0 +1,32 @@
+"""TTS backend interface."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, AsyncIterator, Optional
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from ..voices.profile import VoiceProfile
+
+
+@dataclass
+class SynthesisResult:
+    audio: np.ndarray  # float32 mono in [-1, 1]
+    sample_rate: int
+
+
+class TTSBackend(ABC):
+    @abstractmethod
+    def load(self) -> None: ...
+
+    @abstractmethod
+    async def synthesize(self, text: str, voice_profile: VoiceProfile,
+                         language: Optional[str] = None) -> SynthesisResult: ...
+
+    async def synthesize_streaming(self, text: str, voice_profile, language=None
+                                   ) -> AsyncIterator[SynthesisResult]:
+        """Default streaming = chunk the full synthesis. Override for true streaming."""
+        result = await self.synthesize(text, voice_profile, language)
+        yield result
diff --git a/src/javis/tts/f5_backend.py b/src/javis/tts/f5_backend.py
new file mode 100644
index 0000000..6b8f84e
--- /dev/null
+++ b/src/javis/tts/f5_backend.py
@@ -0,0 +1,152 @@
+"""F5-TTS backend — zero-shot voice cloning via Flow Matching.
+
+Install:  pip install f5-tts
+Model weights are automatically downloaded from Hugging Face on first run.
+License: code = MIT, model weights = CC-BY-NC-4.0.
+"""
+from __future__ import annotations
+
+import asyncio
+import threading
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from .base import SynthesisResult, TTSBackend
+
+log = get_logger("tts.f5")
+
+# F5-TTS native output sample rate.
+_F5_SAMPLE_RATE = 24_000
+
+
+def _pick_ref_clip(clips: list[Path]) -> Optional[Path]:
+    """Return the best reference clip from *clips* for F5-TTS.
+
+    F5-TTS works best with a 3–15 second reference sample in a quiet
+    environment.  We prefer clips closest to 10 seconds and not exceeding
+    20 seconds, falling back to the first clip if none fit the range.
+    """
+    if not clips:
+        return None
+    try:
+        import soundfile as sf
+        scored: list[tuple[float, Path]] = []
+        for clip in clips:
+            try:
+                info = sf.info(str(clip))
+                dur = info.duration
+                # Prefer ~10 s; penalise clips shorter than 3 s or longer than 20 s.
+                if dur < 1.0 or dur > 30.0:
+                    continue
+                score = abs(dur - 10.0)
+                scored.append((score, clip))
+            except Exception:
+                continue
+        if scored:
+            scored.sort(key=lambda t: t[0])
+            return scored[0][1]
+    except ImportError:
+        pass
+    return clips[0]
+
+
+def _load_ref_text(wav_path: Path) -> str:
+    """Load the reference transcript from a sibling .txt file, if present.
+
+    If no transcript file exists, return "" to let F5-TTS auto-transcribe
+    the clip (adds ~1 s latency on first call due to internal ASR).
+    """
+    txt = wav_path.with_suffix(".txt")
+    if txt.exists():
+        return txt.read_text(encoding="utf-8").strip()
+    return ""
+
+
+class F5Backend(TTSBackend):
+    """F5-TTS voice-cloning backend.
+
+    Drop-in TTS backend using F5-TTS.  Configure via::
+
+        tts:
+          backend: f5
+    """
+
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        self._client = None
+        self._lock = threading.Lock()
+
+    # ── public interface ──────────────────────────────────────────────────────
+
+    def load(self) -> None:
+        if self._client is not None:
+            return
+        try:
+            from f5_tts.api import F5TTS
+        except ImportError as exc:
+            raise RuntimeError(
+                "F5-TTS is not installed.  Run:  pip install f5-tts\n"
+                "Then restart Javis.  Model weights (~900 MB) will be downloaded "
+                "from Hugging Face on the first synthesis call."
+            ) from exc
+
+        device = self.config.resolve_device(self.config.tts.device)
+        log.info(f"loading F5-TTS on {device} …")
+        # F5TTS_v1_Base is the current production checkpoint (March 2025).
+        self._client = F5TTS(model_type="F5TTS", device=device)
+        log.info("F5-TTS ready")
+
+    async def synthesize(
+        self,
+        text: str,
+        voice_profile,
+        language: Optional[str] = None,
+    ) -> SynthesisResult:
+        if self._client is None:
+            await asyncio.to_thread(self.load)
+        return await asyncio.to_thread(self._synthesize_sync, text, voice_profile)
+
+    # ── private ───────────────────────────────────────────────────────────────
+
+    def _synthesize_sync(self, text: str, voice_profile) -> SynthesisResult:
+        ref_clip = _pick_ref_clip(voice_profile.reference_clips)
+        if ref_clip is None:
+            raise RuntimeError(
+                f"Voice profile '{voice_profile.name}' has no reference clips.  "
+                "Record at least one WAV in the Voices tab and retry."
+            )
+        ref_text = _load_ref_text(ref_clip)
+
+        speed = (
+            voice_profile.speed
+            if voice_profile.speed is not None
+            else self.config.tts.speed
+        )
+
+        log.debug(
+            f"F5-TTS synthesize: voice={voice_profile.name}, "
+            f"ref={ref_clip.name}, speed={speed:.2f}, text={text[:60]!r}"
+        )
+
+        with self._lock:
+            wav, sr, _ = self._client.infer(
+                ref_file=str(ref_clip),
+                ref_text=ref_text,   # "" triggers built-in auto-transcription
+                gen_text=text,
+                speed=speed,
+                remove_silence=True,
+                seed=-1,
+            )
+
+        # wav is a numpy float32 array at sr (24000 Hz).
+        audio = np.asarray(wav, dtype=np.float32)
+        # Normalise to [-1, 1] in case the model returns slightly clipped audio.
+        peak = np.abs(audio).max()
+        if peak > 1e-6:
+            audio = audio / peak * 0.95
+
+        return SynthesisResult(audio=audio, sample_rate=sr)
diff --git a/src/javis/tts/factory.py b/src/javis/tts/factory.py
new file mode 100644
index 0000000..4dcfd36
--- /dev/null
+++ b/src/javis/tts/factory.py
@@ -0,0 +1,19 @@
+"""Backend factory."""
+from __future__ import annotations
+
+from ..core.config import Config
+from .base import TTSBackend
+
+
+def make_backend(config: Config) -> TTSBackend | None:
+    """Return a TTS backend, or ``None`` if TTS is disabled."""
+    if not config.tts.enabled or config.tts.backend.lower() in ("none", "off", "disabled"):
+        return None
+    name = config.tts.backend.lower()
+    if name == "piper":
+        from .piper_backend import PiperBackend
+        return PiperBackend(config)
+    if name in ("f5", "f5tts", "f5-tts"):
+        from .f5_backend import F5Backend
+        return F5Backend(config)
+    raise ValueError(f"Unknown TTS backend: {config.tts.backend}")
diff --git a/src/javis/tts/models.py b/src/javis/tts/models.py
new file mode 100644
index 0000000..fdb8660
--- /dev/null
+++ b/src/javis/tts/models.py
@@ -0,0 +1,94 @@
+"""Discover Piper ONNX voice models inside the voices directory.
+
+Each trained voice profile is expected to contain ``piper_model.onnx``
+(and an optional ``piper_model.onnx.json`` config).  This module provides
+lightweight helpers for the startup dialog and GUI status panels.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class TTSModelInfo:
+    voice_name: str           # profile directory name
+    display_name: str
+    backend: str = "piper"
+    model_path: Path | None = None
+    size_bytes: int = 0
+    ready: bool = False
+    training_count: int = 0   # utterances in metadata.csv
+
+    @property
+    def size_human(self) -> str:
+        n = float(self.size_bytes)
+        for unit in ("B", "KB", "MB"):
+            if n < 1024:
+                return f"{n:.1f} {unit}"
+            n /= 1024
+        return f"{n:.1f} GB"
+
+    @property
+    def status_text(self) -> str:
+        if self.ready:
+            return f"ready · {self.size_human}"
+        return "not trained"
+
+
+def _count_lines(path: Path) -> int:
+    try:
+        return sum(1 for _ in path.open(encoding="utf-8", errors="ignore"))
+    except OSError:
+        return 0
+
+
+def list_models(voices_dir: Path) -> list[TTSModelInfo]:
+    """Return one entry per voice profile that has a ``piper_model.onnx``."""
+    models: list[TTSModelInfo] = []
+    if not voices_dir.exists():
+        return models
+    for profile_dir in sorted(voices_dir.iterdir()):
+        if not profile_dir.is_dir():
+            continue
+        model_path = profile_dir / "piper_model.onnx"
+        size = 0
+        ready = model_path.exists()
+        if ready:
+            try:
+                size = model_path.stat().st_size
+            except OSError:
+                pass
+        count = _count_lines(profile_dir / "training" / "metadata.csv")
+        models.append(TTSModelInfo(
+            voice_name=profile_dir.name,
+            display_name=profile_dir.name,
+            model_path=model_path if ready else None,
+            size_bytes=size,
+            ready=ready,
+            training_count=count,
+        ))
+    return models
+
+
+def get_model_info(voices_dir: Path, voice_name: str) -> TTSModelInfo:
+    """Return status info for a single voice profile."""
+    profile_dir = voices_dir / voice_name
+    model_path = profile_dir / "piper_model.onnx"
+    ready = model_path.exists()
+    size = 0
+    if ready:
+        try:
+            size = model_path.stat().st_size
+        except OSError:
+            pass
+    count = _count_lines(profile_dir / "training" / "metadata.csv")
+    return TTSModelInfo(
+        voice_name=voice_name,
+        display_name=voice_name,
+        model_path=model_path if ready else None,
+        size_bytes=size,
+        ready=ready,
+        training_count=count,
+    )
+
diff --git a/src/javis/tts/piper_backend.py b/src/javis/tts/piper_backend.py
new file mode 100644
index 0000000..014b9a2
--- /dev/null
+++ b/src/javis/tts/piper_backend.py
@@ -0,0 +1,160 @@
+"""Piper TTS inference backend.
+
+Wraps ``piper-tts`` (``pip install piper-tts``) to provide a fast,
+CPU-capable ONNX TTS backend.
+
+A trained Piper model consists of two files:
+
+    piper_model.onnx        — ONNX weights
+    piper_model.onnx.json   — model config (phonemes, sample rate, …)
+
+The model path is resolved in this order:
+  1. ``config.tts.piper_model`` (explicit path in config.yaml)
+  2. ``{voices_dir}/{tts.default_voice}/piper_model.onnx`` (active voice dir)
+
+Inference is synchronous (ONNX runtime) so it runs in a thread-pool executor
+to avoid blocking the asyncio event loop.
+
+CUDA support: install ``onnxruntime-gpu`` instead of ``onnxruntime`` and Piper
+will use CUDA automatically.
+"""
+from __future__ import annotations
+
+import asyncio
+import re
+import threading
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from .base import SynthesisResult, TTSBackend
+
+log = get_logger("tts.piper")
+
+# Markdown patterns that Piper would speak literally.
+_MD_PATTERNS: list[tuple] = [
+    (re.compile(r"```[\s\S]*?```"), ""),
+    (re.compile(r"`([^`]+)`"), r"\1"),
+    (re.compile(r"\*{1,3}([^*]*)\*{1,3}"), r"\1"),
+    (re.compile(r"_{1,2}([^_]*)_{1,2}"), r"\1"),
+    (re.compile(r"^#{1,6}\s+", re.MULTILINE), ""),
+    (re.compile(r"\[([^\]]+)\]\([^)]+\)"), r"\1"),
+    (re.compile(r"^>\s+", re.MULTILINE), ""),
+    (re.compile(r"^\d+\.\s+", re.MULTILINE), ""),
+    (re.compile(r"^[-*+]\s+", re.MULTILINE), ""),
+]
+_PUNCT_ONLY = re.compile(r"^[^\w\s]+$")
+
+
+def _clean(text: str) -> str:
+    for pat, repl in _MD_PATTERNS:
+        text = pat.sub(repl, text)
+    text = re.sub(r"  +", " ", text).strip()
+    return "" if _PUNCT_ONLY.match(text) else text
+
+
+class PiperBackend(TTSBackend):
+    """Piper ONNX TTS backend."""
+
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        self._voice: Optional[object] = None   # piper.voice.PiperVoice
+        self._sample_rate: int = 22050
+        self._lock = threading.Lock()
+
+    # ── TTSBackend interface ──────────────────────────────────────────────────
+
+    def load(self) -> None:
+        """Load the ONNX model.  Called once at startup."""
+        try:
+            from piper.voice import PiperVoice  # type: ignore[import]
+        except ImportError as exc:
+            raise RuntimeError(
+                "piper-tts is not installed. Run: pip install piper-tts"
+            ) from exc
+
+        model_path = self._resolve_model_path()
+        config_path = Path(str(model_path) + ".json")
+
+        log.info(f"Loading Piper model: {model_path}")
+        self._voice = PiperVoice.load(
+            str(model_path),
+            config_path=str(config_path) if config_path.exists() else None,
+            use_cuda=False,  # ONNX runtime picks CUDA automatically when available
+        )
+        self._sample_rate = (
+            self._voice.config.sample_rate  # type: ignore[union-attr]
+            if hasattr(self._voice, "config")
+            else 22050
+        )
+        log.info(
+            f"Piper model loaded — sample_rate={self._sample_rate} Hz, "
+            f"language={getattr(getattr(self._voice, 'config', None), 'espeak_voice', '?')}"
+        )
+
+    async def synthesize(
+        self,
+        text: str,
+        voice_profile: object,
+        language: Optional[str] = None,
+    ) -> SynthesisResult:
+        text = _clean(text)
+        if not text:
+            return SynthesisResult(
+                audio=np.zeros(0, dtype=np.float32),
+                sample_rate=self._sample_rate,
+            )
+
+        audio = await asyncio.to_thread(self._synthesize_sync, text)
+        return SynthesisResult(audio=audio, sample_rate=self._sample_rate)
+
+    # ── Internal ──────────────────────────────────────────────────────────────
+
+    def _synthesize_sync(self, text: str) -> np.ndarray:
+        """Run Piper inference synchronously (called from thread-pool)."""
+        if self._voice is None:
+            raise RuntimeError("PiperBackend.load() has not been called.")
+
+        from piper.config import SynthesisConfig  # type: ignore[import]
+
+        syn_config = SynthesisConfig(
+            length_scale=self.config.tts.piper_length_scale,
+            noise_scale=self.config.tts.piper_noise_scale,
+            noise_w_scale=self.config.tts.piper_noise_w,
+        )
+
+        chunks: list[np.ndarray] = []
+        with self._lock:
+            for chunk in self._voice.synthesize(text, syn_config):
+                arr = chunk.audio_float_array
+                if arr is not None and len(arr) > 0:
+                    chunks.append(np.asarray(arr, dtype=np.float32))
+
+        if not chunks:
+            return np.zeros(0, dtype=np.float32)
+        return np.concatenate(chunks)
+
+    def _resolve_model_path(self) -> Path:
+        """Return the ONNX model path from config or voice profile dir."""
+        explicit = self.config.tts.piper_model.strip()
+        if explicit:
+            p = Path(explicit)
+            if p.exists():
+                return p
+            log.warning(f"piper_model path not found: {p}")
+
+        # Fall back to active voice profile directory.
+        voice_name = self.config.tts.default_voice
+        voices_dir = self.config.paths.voices_dir
+        candidate = voices_dir / voice_name / "piper_model.onnx"
+        if candidate.exists():
+            return candidate
+
+        raise FileNotFoundError(
+            f"Piper ONNX model not found. Expected at {candidate} "
+            "or set tts.piper_model in config.yaml. "
+            "Run: python -m javis.voices.piper_train --voice "
+        )
diff --git a/src/javis/tts/runner.py b/src/javis/tts/runner.py
new file mode 100644
index 0000000..83d03fb
--- /dev/null
+++ b/src/javis/tts/runner.py
@@ -0,0 +1,26 @@
+"""One-shot synthesis used by `javis --speak`."""
+from __future__ import annotations
+
+from typing import Optional
+
+from ..audio_io.stream import play_pcm
+from ..core.config import Config
+from ..core.logger import get_logger
+from ..voices.manager import VoiceManager
+from .factory import make_backend
+
+log = get_logger("tts.runner")
+
+
+async def speak_once(config: Config, text: str, voice: Optional[str]) -> int:
+    backend = make_backend(config)
+    if backend is None:
+        log.error("TTS is disabled in config — cannot speak.")
+        return 1
+    mgr = VoiceManager(config)
+    name = voice or config.tts.default_voice
+    profile = mgr.get_or_create_default(name)
+    log.info(f"speaking with voice '{profile.name}'")
+    result = await backend.synthesize(text, profile)
+    await play_pcm(result.audio, result.sample_rate, config.audio.output_device)
+    return 0
diff --git a/src/javis/voices/__init__.py b/src/javis/voices/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/javis/voices/audiobook_ingest.py b/src/javis/voices/audiobook_ingest.py
new file mode 100644
index 0000000..5a392d4
--- /dev/null
+++ b/src/javis/voices/audiobook_ingest.py
@@ -0,0 +1,794 @@
+"""Audiobook → cleaned training/reference dataset.
+
+Pipeline:
+  1. Load audio at 22.05 kHz mono.
+  2. silero-vad segments into utterances.
+  3. Keep 3–12 s segments with adequate energy.
+  4. Save short ones as reference clips for zero-shot cloning.
+  5. Transcribe the rest with faster-whisper → LJSpeech-style metadata.csv
+     ready for Piper VITS training.
+
+Checkpointing / resume
+----------------------
+After every ``_CHECKPOINT_SAVE_INTERVAL`` segments (and whenever the caller
+signals a stop) the pipeline saves a JSON checkpoint inside the voice profile
+directory.  On the next call with the same source file the pipeline can resume
+from the last saved position, skipping already-processed segments and reusing
+the VAD output (which can take several minutes on long audiobooks).
+
+Pass ``stop_event`` (an :class:`asyncio.Event`) to let callers request a
+graceful stop.  The pipeline saves the checkpoint and returns with
+``phase="stopped"`` instead of ``phase="done"``.
+
+Pass ``resume=False`` to ignore any existing checkpoint and start fresh.
+"""
+from __future__ import annotations
+
+import asyncio
+import csv
+import json
+import os
+import shutil
+import sys
+from dataclasses import dataclass, field, asdict
+from fractions import Fraction
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import scipy.signal
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from ..stt.transcriber import Transcriber
+
+log = get_logger("voices.ingest")
+
+REF_MIN_S = 5.0
+REF_MAX_S = 12.0
+TRAIN_MIN_S = 2.0
+TRAIN_MAX_S = 15.0
+TARGET_SR = 22050
+
+# ── Narrator-voice quality thresholds for reference clip selection ─────────────
+# These defaults target a warm male baritone narrator recording (e.g. Rufus Beck
+# reading German Harry Potter), where the same reader performs 30+ wildly
+# different character voices.  The filters keep only clips that fall within the
+# narrator's expected F0 band and have speech-like dynamics — rejecting music
+# stingers, extreme character voices (Dobby ≈ 300 Hz, Voldemort ≈ 70 Hz),
+# and multi-voice transitions.
+#
+# The F0 band is speaker-specific, so it is configurable via environment
+# variables for cloning other voices (e.g. a female narrator):
+#   JAVIS_INGEST_F0_MIN / _MAX / _STD_MAX   — override the Hz band / stability
+#   JAVIS_INGEST_F0_GATE=0                   — disable the pitch gate entirely
+# The amplitude, RMS-CV and HF-ratio gates are speaker-agnostic and always on.
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ[name])
+    except (KeyError, ValueError):
+        return default
+
+
+REF_NARRATOR_F0_MIN: float = _env_float("JAVIS_INGEST_F0_MIN", 80.0)    # Hz — lower bound for narrator F0
+REF_NARRATOR_F0_MAX: float = _env_float("JAVIS_INGEST_F0_MAX", 185.0)   # Hz — upper bound for narrator F0
+REF_NARRATOR_F0_STD_MAX: float = _env_float("JAVIS_INGEST_F0_STD_MAX", 42.0)  # Hz — max std of voiced F0 frames
+# Pitch gate is on by default (tuned for the baritone-narrator use case); set
+# JAVIS_INGEST_F0_GATE=0 to skip it when cloning a different voice.
+F0_GATE_ENABLED: bool = os.environ.get("JAVIS_INGEST_F0_GATE", "1").strip().lower() not in ("0", "false", "no", "off")
+REF_MAX_RMS_CV: float = 0.55         # max coefficient of variation of per-window RMS
+REF_MAX_HF_RATIO: float = 0.10       # max fraction of spectral energy above 6 kHz
+REF_MIN_PEAK_AMP: float = 0.04       # minimum peak amplitude (energy gate)
+
+_CHECKPOINT_FILE = "ingest_checkpoint.json"
+# Segments (VAD output) are stored separately as a compact numpy array so the
+# checkpoint JSON never grows with audiobook length.  The .npy file is written
+# once after segmentation and deleted together with the checkpoint on completion.
+_SEGMENTS_FILE = "ingest_segments.npy"
+# Source audio is transcoded once to a flat PCM WAV (mono, TARGET_SR) so that
+# VAD and per-clip reads never decode compressed audio again.
+_CACHE_RAW = "ingest_audio_cache.i16"  # raw int16 LE, mono, TARGET_SR Hz
+# Save checkpoint every N processed segments (not iterations — skipped ones
+# don't count) to limit how much work is lost on an unexpected stop.
+_CHECKPOINT_SAVE_INTERVAL = 20
+
+
+@dataclass
+class IngestStats:
+    total_segments: int = 0
+    reference_clips: int = 0
+    training_clips: int = 0
+    rejected: int = 0
+
+
+# ── checkpoint helpers ────────────────────────────────────────────────────────
+
+@dataclass
+class _Checkpoint:
+    source: str
+    source_size: int
+    voice_name: str
+    next_index: int         # resume the loop from here
+    stats: dict             # IngestStats serialised
+    rows: list              # [[stem, text], …] accumulated so far
+    max_reference_clips: int
+    transcribe: bool
+    saved_at: str = ""
+    clip_offset: int = 0    # first clip_{N}.wav index for this source
+    source_sr: int = 0      # source file sample rate (0 = unknown; re-read from file)
+    # legacy field — segments are now stored in ingest_segments.npy; kept so
+    # that old checkpoint files can still be loaded without a KeyError.
+    segments: list = field(default_factory=list)
+
+
+def _checkpoint_path(prof_dir: Path) -> Path:
+    return prof_dir / _CHECKPOINT_FILE
+
+
+def _save_segments(prof_dir: Path, segments: list[tuple[int, int]]) -> None:
+    """Persist *segments* as a compact int64 numpy array."""
+    arr = np.array(segments, dtype=np.int64)
+    np.save(str(prof_dir / _SEGMENTS_FILE), arr)
+
+
+def _load_segments(prof_dir: Path) -> Optional[list[tuple[int, int]]]:
+    """Load segments written by :func:`_save_segments`, or return ``None``."""
+    p = prof_dir / _SEGMENTS_FILE
+    if not p.exists():
+        return None
+    try:
+        arr = np.load(str(p))
+        return [(int(a), int(b)) for a, b in arr]
+    except Exception as exc:
+        log.debug(f"segments file load failed ({exc}) — ignoring")
+        return None
+
+
+def _load_checkpoint(prof_dir: Path, source: Path) -> Optional[_Checkpoint]:
+    """Return a matching checkpoint, or ``None`` if none exists / source differs."""
+    p = _checkpoint_path(prof_dir)
+    if not p.exists():
+        return None
+    try:
+        with open(p, encoding="utf-8") as f:
+            data = json.load(f)
+        # Validate the checkpoint belongs to the same source file.
+        if data.get("source_size") != source.stat().st_size:
+            return None
+        data.setdefault("clip_offset", 0)  # backward compat with older checkpoints
+        data.setdefault("segments", [])    # backward compat: may be absent in new format
+        return _Checkpoint(**data)
+    except Exception as exc:
+        log.debug(f"checkpoint load failed ({exc}) — ignoring")
+        return None
+
+
+def _save_checkpoint(prof_dir: Path, cp: _Checkpoint) -> None:
+    import time as _t
+    cp.saved_at = _t.strftime("%Y-%m-%dT%H:%M:%S")
+    p = _checkpoint_path(prof_dir)
+    tmp = p.with_suffix(".json.part")
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(asdict(cp), f, ensure_ascii=False)
+        tmp.replace(p)
+    except Exception as exc:
+        log.warning(f"checkpoint save failed: {exc}")
+
+
+def _delete_checkpoint(prof_dir: Path) -> None:
+    for name in (_CHECKPOINT_FILE, _SEGMENTS_FILE, _CACHE_RAW):
+        try:
+            (prof_dir / name).unlink(missing_ok=True)
+        except Exception:
+            pass
+
+
+def checkpoint_info(config: Config, voice_name: str, source: Path) -> Optional[dict]:
+    """Return a summary dict for an existing checkpoint, or ``None``.
+
+    Called by the GUI to decide whether to offer a resume prompt.
+    Returns ``{"next_index": N, "total": M, "saved_at": "..."}`` when found.
+    """
+    prof_dir = config.paths.voices_dir / voice_name
+    cp = _load_checkpoint(prof_dir, source)
+    if cp is None:
+        return None
+    # Segment count comes from the .npy file (new format) or the legacy inline list.
+    segs = _load_segments(prof_dir)
+    total = len(segs) if segs is not None else len(cp.segments)
+    return {
+        "next_index": cp.next_index,
+        "total": total,
+        "saved_at": cp.saved_at,
+    }
+
+
+def _is_clean_ref_clip(clip: np.ndarray, sr: int) -> bool:
+    """Return True if *clip* looks like a clean single-voice narrator segment.
+
+    Applies three fast quality gates in order of increasing cost:
+
+    1. **Peak amplitude gate** — rejects near-silence or extremely low-energy
+       clips (indicates noise floor or a very quiet passage).
+
+    2. **RMS coefficient-of-variation gate** — rejects clips with highly
+       uneven loudness across 150 ms windows.  A high CV indicates sound
+       effects, music transitions, or rapid switches between very different
+       character voices.
+
+    3. **High-frequency energy ratio gate** *(librosa)* — background music
+       and heavy sound effects contain disproportionately high energy above
+       6 kHz relative to clean speech.
+
+    4. **Pitch-range + stability gate** *(librosa.yin)* — checks that the
+       voiced F0 of the clip falls within ``REF_NARRATOR_F0_MIN``–
+       ``REF_NARRATOR_F0_MAX`` Hz and that its standard deviation is below
+       ``REF_NARRATOR_F0_STD_MAX``.  Extreme character voices (e.g. very
+       high-pitched, very low / breathy) are rejected here.
+
+    Gates 3 and 4 are wrapped in a broad ``except`` so that a missing
+    optional dependency never blocks ingest.
+    """
+    # Gate 1: peak amplitude
+    if float(np.max(np.abs(clip))) < REF_MIN_PEAK_AMP:
+        return False
+
+    # Gate 2: RMS coefficient of variation over 150 ms windows
+    hop = max(1, int(sr * 0.15))
+    n_win = len(clip) // hop
+    if n_win >= 3:
+        rms_vals = np.array([
+            np.sqrt(np.mean(clip[i * hop:(i + 1) * hop] ** 2))
+            for i in range(n_win)
+        ], dtype=np.float32)
+        mean_rms = float(np.mean(rms_vals))
+        if mean_rms > 1e-6:
+            rms_cv = float(np.std(rms_vals)) / mean_rms
+            if rms_cv > REF_MAX_RMS_CV:
+                return False
+
+    # Gates 3 & 4: spectral and pitch analysis via librosa (optional)
+    try:
+        import librosa  # noqa: PLC0415
+
+        # Gate 3: high-frequency energy fraction
+        spec = np.abs(librosa.stft(clip, n_fft=2048, hop_length=512))
+        freqs = librosa.fft_frequencies(sr=sr, n_fft=2048)
+        hf_mask = freqs > 6000
+        total_energy = float(np.sum(spec ** 2)) + 1e-10
+        hf_energy = float(np.sum(spec[hf_mask, :] ** 2))
+        if hf_energy / total_energy > REF_MAX_HF_RATIO:
+            return False
+
+        # Gate 4: voiced F0 range and stability.  Speaker-specific — skip when
+        # disabled via JAVIS_INGEST_F0_GATE so non-baritone voices aren't rejected.
+        # IMPORTANT: fmax must be well ABOVE the rejection threshold so that
+        # out-of-band pitches (Dobby ≈ 300-350 Hz) are detected as-is rather
+        # than being aliased down by YIN to a harmonic within the acceptance band.
+        if not F0_GATE_ENABLED:
+            return True
+        analysis_fmax = min(float(sr) / 2.0, 600.0)
+        analysis_fmin = max(20.0, REF_NARRATOR_F0_MIN * 0.6)
+        try:
+            f0, voiced_flag, _ = librosa.pyin(
+                clip,
+                fmin=analysis_fmin,
+                fmax=analysis_fmax,
+                sr=sr,
+                hop_length=512,
+                fill_na=0.0,
+            )
+            voiced_f0 = f0[voiced_flag & (f0 > 0)]
+        except Exception:
+            f0 = librosa.yin(
+                clip,
+                fmin=analysis_fmin,
+                fmax=analysis_fmax,
+                sr=sr,
+                hop_length=512,
+            )
+            voiced_f0 = f0[f0 > 0]
+
+        if len(voiced_f0) == 0:
+            return False  # no voiced content detected
+
+        in_band = voiced_f0[
+            (voiced_f0 >= REF_NARRATOR_F0_MIN) & (voiced_f0 <= REF_NARRATOR_F0_MAX)
+        ]
+        # At least 50 % of voiced F0 frames must be within the narrator band
+        if len(in_band) < len(voiced_f0) * 0.50:
+            return False
+        # Voiced frames within the band must be relatively stable
+        if len(in_band) >= 6 and float(np.std(in_band)) > REF_NARRATOR_F0_STD_MAX:
+            return False
+
+    except Exception:
+        pass  # librosa unavailable or analysis error — skip spectral/pitch gates
+
+    return True
+
+
+def _load_audio(path: Path, target_sr: int = TARGET_SR) -> np.ndarray:
+    # soundfile 0.12+ ships libsndfile 1.2.2 which reads MP3 natively.
+    # OGG Vorbis and FLAC were always supported.
+    # M4A / AAC / OPUS / WMA are not supported by libsndfile — convert to
+    # MP3 or WAV first.
+    import soundfile as sf  # type: ignore[import]
+
+    info = sf.info(str(path))
+    sr = info.samplerate
+
+    # MP3 files with malformed ID3 tags (e.g. non-syncsafe CTOC frames) cause
+    # libsndfile to report INT64_MAX for the frame count.  Calling sf.read()
+    # then tries to pre-allocate that many samples and raises a ValueError.
+    # Fall back to chunked streaming via sf.blocks() which reads until EOF
+    # without pre-allocating based on the (bogus) total-frame count.
+    #
+    # Memory note: we use a small blocksize (10 s) and resample + mono-mix
+    # each block immediately so that only the TARGET_SR data accumulates
+    # (roughly 2× smaller than keeping raw source-rate stereo chunks).
+    if info.frames >= sys.maxsize // 2:
+        log.warning(f"_load_audio: unknown frame count for {path.name} — using chunked read")
+        frac = Fraction(target_sr, sr).limit_denominator(1000)
+        resampled_chunks: list[np.ndarray] = []
+        for block in sf.blocks(str(path), blocksize=sr * 10, dtype="float32", always_2d=True):
+            # Mono-mix immediately so we don't carry stereo data any longer than needed.
+            mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
+            chunk = (
+                scipy.signal.resample_poly(mono, frac.numerator, frac.denominator).astype(np.float32)
+                if sr != target_sr else mono.astype(np.float32)
+            )
+            resampled_chunks.append(chunk)
+            del block, mono  # free source-rate data immediately
+        if not resampled_chunks:
+            raise ValueError(f"No audio data read from {path}")
+        audio: np.ndarray = np.concatenate(resampled_chunks)
+        del resampled_chunks
+        return audio  # already mono, already at target_sr
+    else:
+        audio, _ = sf.read(str(path), dtype="float32", always_2d=True)
+
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+
+    if sr != target_sr:
+        frac = Fraction(target_sr, sr).limit_denominator(1000)
+        audio = scipy.signal.resample_poly(audio, frac.numerator, frac.denominator).astype(np.float32)
+    return audio
+
+
+def _segment(audio: np.ndarray, sr: int) -> list[tuple[int, int]]:
+    """Return list of (start_sample, end_sample) speech segments via silero-vad."""
+    from silero_vad import load_silero_vad, get_speech_timestamps  # heavy optional dep
+    import torch  # heavy optional dep
+    model = load_silero_vad()
+    # silero-vad expects 16k; resample and translate sample offsets back.
+    frac16 = Fraction(16000, sr).limit_denominator(1000)
+    audio16 = scipy.signal.resample_poly(audio, frac16.numerator, frac16.denominator).astype(np.float32)
+    timestamps = get_speech_timestamps(
+        torch.from_numpy(audio16), model, sampling_rate=16000, return_seconds=False,
+        min_silence_duration_ms=300, min_speech_duration_ms=500,
+    )
+    out = []
+    ratio = sr / 16000.0
+    for t in timestamps:
+        out.append((int(t["start"] * ratio), int(t["end"] * ratio)))
+    return out
+
+
+def _save_wav(path: Path, audio: np.ndarray, sr: int) -> None:
+    import soundfile as sf
+    path.parent.mkdir(parents=True, exist_ok=True)
+    sf.write(str(path), audio, sr, subtype="PCM_16")
+
+
+def _transcode_to_raw(source: Path, target: Path, target_sr: int) -> None:
+    """Stream *source* → raw int16 LE binary (mono, *target_sr* Hz).
+
+    Peak RAM: one 10-second block (~4 MiB).  No file-size limit.
+    Writes atomically via a .part temp file.
+    """
+    import soundfile as sf
+
+    info = sf.info(str(source))
+    sr = info.samplerate
+    frac = Fraction(target_sr, sr).limit_denominator(1000)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    tmp = target.with_suffix(".part")
+
+    # Pre-check disk space when duration is known (skip for malformed MP3s with
+    # bogus INT64_MAX frame counts).
+    if info.frames < sys.maxsize // 2:
+        estimated_bytes = int(info.frames / sr * target_sr) * 2  # int16 = 2 B
+        free = shutil.disk_usage(str(target.parent)).free
+        if free < estimated_bytes + 64 * 1024 * 1024:
+            raise RuntimeError(
+                f"Not enough disk space to cache ‘{source.name}’: "
+                f"need ~{estimated_bytes / 1_000_000_000:.1f} GB, "
+                f"have {free / 1_000_000_000:.1f} GB free on {target.parent}"
+            )
+
+    try:
+        with open(str(tmp), "wb") as out_f:
+            for block in sf.blocks(str(source), blocksize=sr * 10,
+                                    dtype="float32", always_2d=True):
+                mono = block.mean(axis=1) if block.ndim > 1 else block.ravel()
+                if sr != target_sr:
+                    chunk = scipy.signal.resample_poly(
+                        mono, frac.numerator, frac.denominator
+                    )
+                else:
+                    chunk = mono
+                i16 = np.clip(chunk * 32767.0, -32768, 32767).astype(np.int16)
+                out_f.write(i16.tobytes())
+                del block, mono, chunk, i16
+        tmp.replace(target)
+    except Exception:
+        try:
+            tmp.unlink(missing_ok=True)
+        except Exception:
+            pass
+        raise
+
+
+def _segment_from_raw(raw_path: Path, target_sr: int) -> list[tuple[int, int]]:
+    """Segment a raw int16 cache file using silero-VAD in 10-minute chunks.
+
+    Peak RAM per chunk: ~90 MiB (22 kHz float32 block + 16 kHz float32 block).
+    The full-length audio is never loaded into RAM at once.
+    """
+    from silero_vad import load_silero_vad, get_speech_timestamps
+    import torch
+
+    model = load_silero_vad()
+    frac16 = Fraction(16000, target_sr).limit_denominator(1000)
+    CHUNK_22K = target_sr * 600  # 10-minute window at TARGET_SR
+
+    mm = np.memmap(str(raw_path), dtype=np.int16, mode="r")
+    total_22k = len(mm)
+    segments: list[tuple[int, int]] = []
+
+    offset = 0
+    while offset < total_22k:
+        end = min(offset + CHUNK_22K, total_22k)
+        chunk_f32 = mm[offset:end].astype(np.float32) / 32767.0
+        audio16 = scipy.signal.resample_poly(
+            chunk_f32, frac16.numerator, frac16.denominator
+        ).astype(np.float32)
+        del chunk_f32
+
+        ts = get_speech_timestamps(
+            torch.from_numpy(audio16),
+            model,
+            sampling_rate=16000,
+            return_seconds=False,
+            min_silence_duration_ms=300,
+            min_speech_duration_ms=500,
+        )
+        del audio16
+
+        ratio = target_sr / 16000.0
+        for t in ts:
+            segments.append((
+                offset + int(t["start"] * ratio),
+                offset + int(t["end"] * ratio),
+            ))
+        offset = end
+
+    del mm
+    return segments
+
+
+def _read_clip_from_raw(raw_path: Path, start: int, end: int) -> np.ndarray:
+    """Read samples [*start*, *end*) from a raw int16 cache.  O(1) seek."""
+    mm = np.memmap(str(raw_path), dtype=np.int16, mode="r")
+    clip = mm[start:end].astype(np.float32) / 32767.0
+    del mm
+    return clip
+
+
+async def ingest_audiobook(
+    config: Config,
+    source: Path,
+    voice_name: str,
+    max_reference_clips: int = 9999,
+    transcribe: bool = True,
+    progress_cb: Optional[callable] = None,  # type: ignore[type-arg]
+    bus: Optional[object] = None,
+    task_id: Optional[str] = None,
+    stop_event: Optional[asyncio.Event] = None,
+    resume: bool = True,
+) -> IngestStats:
+    """Process *source* (any sf-readable audio) into the voice profile *voice_name*.
+
+    If *bus* is an EventBus, emit ``ingest_status`` events for the GUI.
+
+    Parameters
+    ----------
+    stop_event:
+        When set, the pipeline saves a checkpoint and returns early with
+        ``phase="stopped"``.  Pass an :class:`asyncio.Event` created by the
+        caller so the GUI (or any other controller) can request a graceful stop.
+    resume:
+        When ``True`` (default) and a checkpoint for *source* exists inside the
+        profile directory, the pipeline resumes from the saved position.
+        When ``False`` the checkpoint is ignored and ingestion starts fresh.
+    """
+    from .manager import VoiceManager
+
+    async def _emit(payload: dict) -> None:
+        if bus is None:
+            return
+        try:
+            payload = {"id": task_id or voice_name, "voice": voice_name, **payload}
+            await bus.publish("ingest_status", payload)  # type: ignore[attr-defined]
+        except Exception:
+            pass
+
+    mgr = VoiceManager(config)
+    if (mgr.root / voice_name).exists():
+        prof = mgr.get(voice_name)
+    else:
+        prof = mgr.create(voice_name)
+    assert prof.dir is not None
+
+    refs_dir = prof.dir / "refs"
+    train_dir = prof.dir / "training" / "wavs"
+    metadata_csv = prof.dir / "training" / "metadata.csv"
+    refs_dir.mkdir(parents=True, exist_ok=True)
+    train_dir.mkdir(parents=True, exist_ok=True)
+
+    # ── checkpoint / resume ──────────────────────────────────────────────────
+    cp: Optional[_Checkpoint] = None
+    start_index = 0
+    rows: list[tuple[str, str]] = []
+    stats = IngestStats()
+
+    if resume:
+        cp = _load_checkpoint(prof.dir, source)
+
+    # ── ref naming offset ─────────────────────────────────────────────────────
+    # Fresh runs start their ref_NNN.wav numbering after the highest index that
+    # already exists in refs/, so clips from previous ingest runs are never
+    # overwritten.  Resumed runs restore stats.reference_clips from the
+    # checkpoint, which already encodes the correct in-run count, so their
+    # offset stays at 0.
+    _ref_offset: int = 0
+    if cp is None:
+        for _p in refs_dir.glob("ref_*.wav"):
+            try:
+                _ref_offset = max(_ref_offset, int(_p.stem.split("_")[1]) + 1)
+            except (IndexError, ValueError):
+                pass
+
+    # ── training clip naming offset ───────────────────────────────────────────
+    # Same principle: fresh runs start clip numbering after the highest
+    # existing index so different source files never overwrite each other.
+    _clip_offset: int = 0
+    if cp is None:
+        for _p in train_dir.glob("clip_*.wav"):
+            try:
+                _clip_offset = max(_clip_offset, int(_p.stem.split("_")[1]) + 1)
+            except (IndexError, ValueError):
+                pass
+    else:
+        _clip_offset = cp.clip_offset
+
+    # ── raw audio cache ─────────────────────────────────────────────────────────────────
+    # Source is streamed block-by-block into a flat int16 LE binary (~4 MiB
+    # peak RAM per block, no file-size limit).  VAD and clip reads operate on
+    # this cache via numpy memmap — O(1) seek, ~90 MiB peak during segmentation.
+    cache_raw = prof.dir / _CACHE_RAW
+    if cp is None:
+        # Fresh run — remove any stale cache from a previous source.
+        if cache_raw.exists():
+            try:
+                cache_raw.unlink()
+            except Exception:
+                pass
+        log.info(f"transcoding {source.name} → raw cache ...")
+        await _emit({"phase": "loading", "source": source.name})
+        await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR)
+        _cache_dur = cache_raw.stat().st_size / 2 / TARGET_SR  # int16 = 2 bytes
+        log.info(f"transcoded: {_cache_dur:.1f}s")
+        await _emit({"phase": "loaded", "audio_s": _cache_dur})
+    elif not cache_raw.exists():
+        # Resuming but cache was deleted — retranscode.
+        log.info(f"raw cache missing — retranscoding {source.name} ...")
+        await _emit({"phase": "loading", "source": source.name})
+        await asyncio.to_thread(_transcode_to_raw, source, cache_raw, TARGET_SR)
+        log.info("retranscoded OK")
+
+    if cp is not None:
+        # Segments: try the compact .npy file first (new format), fall back to
+        # the legacy inline list inside the checkpoint JSON (old format).
+        segments_from_file = _load_segments(prof.dir)
+        if segments_from_file is not None:
+            segments = segments_from_file
+        elif cp.segments:
+            segments = [tuple(s) for s in cp.segments]
+        else:
+            log.warning("checkpoint has no segments data — starting fresh")
+            cp = None
+            segments = []
+
+        if cp is not None:
+            log.info(
+                f"resuming ingest from segment {cp.next_index}/{len(segments)} "
+                f"(saved {cp.saved_at})"
+            )
+            await _emit({
+                "phase": "resuming",
+                "from_index": cp.next_index,
+                "total": len(segments),
+                "saved_at": cp.saved_at,
+            })
+            start_index = cp.next_index
+            rows = [tuple(r) for r in cp.rows]
+            stats = IngestStats(**cp.stats)
+
+    if cp is None:
+        # Segment in 10-minute streaming chunks — ~90 MiB peak RAM, never loads
+        # the full audio array.
+        log.info("segmenting via silero-vad (streaming chunks) ...")
+        await _emit({"phase": "segmenting"})
+        segments = await asyncio.to_thread(_segment_from_raw, cache_raw, TARGET_SR)
+        log.info(f"got {len(segments)} segments")
+        await _emit({"phase": "segmented", "segments": len(segments)})
+        stats.total_segments = len(segments)
+        _save_segments(prof.dir, segments)
+
+    transcriber = Transcriber(config) if transcribe else None
+    if transcriber is not None:
+        await _emit({"phase": "loading_stt"})
+        await asyncio.to_thread(transcriber.load)
+
+    last_emit = 0.0
+    processed_since_save = 0
+    import time as _time
+
+    # Slice the segment list so we never iterate over already-processed entries.
+    for i, (start, end) in enumerate(segments[start_index:], start=start_index):
+        # ── stop check ────────────────────────────────────────────────────────
+        if stop_event is not None and stop_event.is_set():
+            log.info(f"ingest stopped by user at segment {i}/{len(segments)}")
+            _save_checkpoint(prof.dir, _Checkpoint(
+                source=str(source),
+                source_size=source.stat().st_size,
+                voice_name=voice_name,
+                next_index=i,
+                stats=asdict(stats),
+                rows=[list(r) for r in rows],
+                max_reference_clips=max_reference_clips,
+                transcribe=transcribe,
+                clip_offset=_clip_offset,
+                source_sr=TARGET_SR,
+            ))
+            _flush_rows(rows, metadata_csv, append=start_index > 0 or metadata_csv.exists())
+            await _emit({
+                "phase": "stopped",
+                "at_index": i, "total": len(segments),
+                "refs": stats.reference_clips, "train": stats.training_clips,
+            })
+            return stats
+
+        # Duration check — fast arithmetic, no I/O.
+        dur = (end - start) / TARGET_SR
+        if dur < TRAIN_MIN_S or dur > TRAIN_MAX_S:
+            stats.rejected += 1
+            continue
+
+        # Read this clip from the raw int16 cache — O(1) memmap seek.
+        clip = await asyncio.to_thread(_read_clip_from_raw, cache_raw, start, end)
+
+        if np.max(np.abs(clip)) < 0.02:
+            stats.rejected += 1
+            continue
+
+        # Reference clip selection — apply narrator quality filter to keep
+        # only clean single-voice segments that match the narrator's F0 range.
+        if REF_MIN_S <= dur <= REF_MAX_S and stats.reference_clips < max_reference_clips:
+            if await asyncio.to_thread(_is_clean_ref_clip, clip, TARGET_SR):
+                ref_path = refs_dir / f"ref_{_ref_offset + stats.reference_clips:03d}.wav"
+                await asyncio.to_thread(_save_wav, ref_path, clip, TARGET_SR)
+                stats.reference_clips += 1
+            else:
+                log.debug(f"segment {i} skipped for ref (quality gate failed, dur={dur:.1f}s)")
+
+        # Training clip
+        clip_path = train_dir / f"clip_{_clip_offset + i:05d}.wav"
+        await asyncio.to_thread(_save_wav, clip_path, clip, TARGET_SR)
+        stats.training_clips += 1
+
+        if transcriber is not None:
+            try:
+                tr = await transcriber.transcribe(clip, TARGET_SR)
+                if tr.text:
+                    rows.append((clip_path.stem, tr.text))
+            except Exception as e:  # noqa: BLE001
+                log.warning(f"transcribe failed for {clip_path.name}: {e}")
+
+        processed_since_save += 1
+        if processed_since_save >= _CHECKPOINT_SAVE_INTERVAL:
+            # Segments are stored in the .npy file; only lightweight metadata
+            # goes in the JSON so checkpoint writes stay O(1) regardless of
+            # audiobook length.
+            _save_checkpoint(prof.dir, _Checkpoint(
+                source=str(source),
+                source_size=source.stat().st_size,
+                voice_name=voice_name,
+                next_index=i + 1,
+                stats=asdict(stats),
+                rows=[list(r) for r in rows],
+                max_reference_clips=max_reference_clips,
+                transcribe=transcribe,
+                clip_offset=_clip_offset,
+                source_sr=TARGET_SR,
+            ))
+            processed_since_save = 0
+
+        if progress_cb:
+            try:
+                progress_cb(i + 1, len(segments))
+            except Exception:
+                pass
+
+        now = _time.monotonic()
+        if now - last_emit > 0.5 or i + 1 == len(segments):
+            last_emit = now
+            await _emit({
+                "phase": "processing",
+                "done": i + 1, "total": len(segments),
+                "refs": stats.reference_clips, "train": stats.training_clips,
+                "rejected": stats.rejected,
+            })
+
+    _flush_rows(rows, metadata_csv, append=start_index > 0 or metadata_csv.exists())
+
+    # Refresh and save profile so the new refs are picked up.
+    prof = mgr.get(voice_name)
+    prof.save()
+
+    _delete_checkpoint(prof.dir)
+    log.info(f"ingest done: {stats}")
+    await _emit({
+        "phase": "done",
+        "refs": stats.reference_clips, "train": stats.training_clips,
+        "rejected": stats.rejected, "total": stats.total_segments,
+    })
+    return stats
+
+
+def _flush_rows(
+    rows: list[tuple[str, str]], metadata_csv: Path, *, append: bool
+) -> None:
+    """Write *rows* to *metadata_csv* in LJSpeech format.
+
+    When *append* is True (resuming a previous run) existing rows are
+    preserved and new ones are appended; duplicates are de-duplicated by stem.
+    """
+    if not rows:
+        return
+
+    if append and metadata_csv.exists():
+        # Read existing rows, merge, de-duplicate by stem (new wins).
+        existing: dict[str, str] = {}
+        try:
+            with open(metadata_csv, encoding="utf-8", newline="") as f:
+                for stem, text, *_ in csv.reader(f, delimiter="|"):
+                    existing[stem] = text
+        except Exception:
+            pass
+        for stem, text in rows:
+            existing[stem] = text
+        rows = list(existing.items())
+
+    metadata_csv.parent.mkdir(parents=True, exist_ok=True)
+    with open(metadata_csv, "w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f, delimiter="|")
+        for stem, text in rows:
+            writer.writerow([stem, text, text])
+    log.info(f"wrote metadata.csv ({len(rows)} rows)")
diff --git a/src/javis/voices/finetune.py b/src/javis/voices/finetune.py
new file mode 100644
index 0000000..700a3db
--- /dev/null
+++ b/src/javis/voices/finetune.py
@@ -0,0 +1,95 @@
+"""Piper VITS training hook.
+
+Delegates to :mod:`javis.voices.piper_train` which wraps the three-stage
+piper-train pipeline (preprocess → VITS train → ONNX export).
+Calling this module directly runs training synchronously; the GUI launches
+it as a background subprocess via ``piper_train.py --voice ``.
+"""
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from .manager import VoiceManager
+
+log = get_logger("voices.finetune")
+
+
+async def finetune_voice(
+    config: Config,
+    voice_name: str,
+    *,
+    language: str = "de",
+    base_checkpoint: str | None = None,
+    batch_size: int = 32,
+    max_epochs: int = 6000,
+) -> int:
+    """Run the full Piper training pipeline for *voice_name*.
+
+    Requires ``piper-train`` and ``espeak-ng`` to be installed.
+    The resulting ONNX model is written to
+    ``{voice_dir}/piper_model.onnx``.
+    """
+    from .piper_train import run_pipeline
+
+    mgr = VoiceManager(config)
+    prof = mgr.get(voice_name)
+    if prof.dir is None:
+        log.error(f"Voice profile '{voice_name}' has no directory.")
+        return 2
+    metadata = prof.dir / "training" / "metadata.csv"
+
+    if not metadata.exists():
+        log.error(
+            f"No training data for '{voice_name}'. Ingest an audiobook first."
+        )
+        return 2
+
+    with metadata.open("r", encoding="utf-8") as f:
+        n_lines = sum(1 for _ in f)
+    log.info(f"'{voice_name}' has {n_lines} training utterances — starting Piper training.")
+
+    rc = await asyncio.to_thread(
+        run_pipeline,
+        prof.dir,
+        language=language,
+        base_checkpoint=Path(base_checkpoint) if base_checkpoint else None,
+        batch_size=batch_size,
+        max_epochs=max_epochs,
+    )
+    return rc
+
+
+def cli() -> int:
+    import argparse
+    from ..core.config import load_config
+    from ..core.logger import setup_logger
+
+    p = argparse.ArgumentParser()
+    p.add_argument("voice")
+    p.add_argument("--language", default="de")
+    p.add_argument("--base-checkpoint", default=None)
+    p.add_argument("--batch-size", type=int, default=32)
+    p.add_argument("--max-epochs", type=int, default=6000)
+    p.add_argument("--config", type=Path, default=Path("config.yaml"))
+    args = p.parse_args()
+
+    config = load_config(args.config)
+    setup_logger(config)
+    return asyncio.run(
+        finetune_voice(
+            config,
+            args.voice,
+            language=args.language,
+            base_checkpoint=args.base_checkpoint,
+            batch_size=args.batch_size,
+            max_epochs=args.max_epochs,
+        )
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(cli())
diff --git a/src/javis/voices/io.py b/src/javis/voices/io.py
new file mode 100644
index 0000000..de02580
--- /dev/null
+++ b/src/javis/voices/io.py
@@ -0,0 +1,269 @@
+"""Import / export voice profiles as portable ``.javisvoice`` zip archives.
+
+A voice profile on disk is a directory tree:
+
+    //
+        profile.json
+        refs/*.wav
+        training/wavs/*.wav            (optional)
+        finetuned/...                  (optional)
+
+An exported archive is a plain ZIP containing those files plus a small
+``MANIFEST.json`` at the top level for forward-compat versioning. Reference
+clip paths are *not* persisted (they are rediscovered from ``refs/`` on load),
+which keeps profiles portable across machines / users.
+"""
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from ..core.logger import get_logger
+from .profile import VoiceProfile
+
+log = get_logger("voices.io")
+
+
+ARCHIVE_SUFFIX = ".javisvoice"
+MANIFEST_NAME = "MANIFEST.json"
+MANIFEST_VERSION = 1
+
+# Files/dirs (relative to the profile root) we pack into the archive.
+_INCLUDE_DIRS = ("refs", "training", "finetuned")
+_INCLUDE_FILES = ("profile.json",)
+
+# Defensive cap to avoid pathological archives (~5 GB).
+_MAX_TOTAL_BYTES = 5 * 1024 * 1024 * 1024
+
+
+@dataclass
+class ExportResult:
+    path: Path
+    profile_name: str
+    files: int
+    bytes: int
+
+
+@dataclass
+class ImportResult:
+    path: Path
+    profile_name: str
+    files: int
+    bytes: int
+    renamed_from: Optional[str] = None
+
+
+def _sanitize_name(name: str) -> str:
+    """Make a string safe to use as a directory name on Windows/macOS/Linux."""
+    cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", name).strip(" .")
+    cleaned = cleaned or "voice"
+    return cleaned[:80]
+
+
+def _safe_extract_target(base: Path, member: str) -> Optional[Path]:
+    """Resolve ``member`` against ``base``, refusing absolute paths and
+    parent-directory traversal (zip-slip)."""
+    if not member or member.endswith("/"):
+        return None  # directory entries are recreated implicitly
+    # Normalize and reject absolute / drive paths.
+    member = member.replace("\\", "/")
+    if member.startswith("/") or re.match(r"^[a-zA-Z]:", member):
+        return None
+    target = (base / member).resolve()
+    try:
+        target.relative_to(base.resolve())
+    except ValueError:
+        return None
+    return target
+
+
+def export_profile(profile_dir: Path, dest: Path) -> ExportResult:
+    """Pack ``profile_dir`` into a ``.javisvoice`` zip at ``dest``.
+
+    ``dest`` may be a directory (filename derived from the profile name) or a
+    full file path. Existing destination files are overwritten.
+    """
+    if not profile_dir.exists() or not profile_dir.is_dir():
+        raise FileNotFoundError(f"voice profile not found: {profile_dir}")
+
+    profile = VoiceProfile.from_dir(profile_dir)
+    name = profile.name or profile_dir.name
+
+    if dest.is_dir() or (not dest.suffix and not dest.exists()):
+        dest.mkdir(parents=True, exist_ok=True)
+        dest = dest / f"{_sanitize_name(name)}{ARCHIVE_SUFFIX}"
+
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dest.with_suffix(dest.suffix + ".part")
+
+    files_written = 0
+    bytes_written = 0
+    try:
+        with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_DEFLATED,
+                             allowZip64=True) as zf:
+            manifest = {
+                "version": MANIFEST_VERSION,
+                "kind": "javis.voice_profile",
+                "name": name,
+                "language": profile.language,
+                "speed": profile.speed,
+                "temperature": profile.temperature,
+                "has_finetuned": profile.finetuned_model is not None,
+            }
+            zf.writestr(MANIFEST_NAME,
+                        json.dumps(manifest, indent=2, ensure_ascii=False))
+
+            for fname in _INCLUDE_FILES:
+                f = profile_dir / fname
+                if f.exists() and f.is_file():
+                    zf.write(f, arcname=fname)
+                    files_written += 1
+                    bytes_written += f.stat().st_size
+
+            for sub in _INCLUDE_DIRS:
+                d = profile_dir / sub
+                if not d.exists():
+                    continue
+                for p in d.rglob("*"):
+                    if not p.is_file():
+                        continue
+                    arc = p.relative_to(profile_dir).as_posix()
+                    zf.write(p, arcname=arc)
+                    files_written += 1
+                    bytes_written += p.stat().st_size
+        tmp.replace(dest)
+    except Exception:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+        raise
+
+    log.info(f"exported voice '{name}' -> {dest} ({files_written} files, {bytes_written} B)")
+    return ExportResult(path=dest, profile_name=name, files=files_written,
+                        bytes=bytes_written)
+
+
+def inspect_archive(archive: Path) -> dict:
+    """Read the manifest of a ``.javisvoice`` archive without extracting.
+
+    Returns at minimum ``{"name": str, "language": str, ...}``. For archives
+    written by an older tool that lacks a manifest, falls back to scanning the
+    archive for a top-level ``profile.json``.
+    """
+    if not archive.exists():
+        raise FileNotFoundError(archive)
+    with zipfile.ZipFile(archive, "r") as zf:
+        names = zf.namelist()
+        if MANIFEST_NAME in names:
+            return json.loads(zf.read(MANIFEST_NAME).decode("utf-8"))
+        # Legacy / unrecognized archive: derive from profile.json if present.
+        for n in names:
+            if n.endswith("profile.json") and n.count("/") <= 1:
+                data = json.loads(zf.read(n).decode("utf-8"))
+                return {
+                    "version": 0,
+                    "kind": "javis.voice_profile",
+                    "name": data.get("name", archive.stem),
+                    "language": data.get("language", "en"),
+                    "has_finetuned": False,
+                }
+    return {"version": 0, "kind": "unknown", "name": archive.stem, "language": "en"}
+
+
+def import_profile(archive: Path, voices_root: Path,
+                   new_name: Optional[str] = None,
+                   overwrite: bool = False) -> ImportResult:
+    """Extract ``archive`` into ``voices_root//``.
+
+    - ``new_name`` overrides the manifest name (sanitized).
+    - If the destination already exists and ``overwrite`` is False, a numeric
+      suffix is appended (``my_voice_2``, ``my_voice_3``, …).
+    - Refuses zip-slip / absolute paths.
+    """
+    if not archive.exists():
+        raise FileNotFoundError(archive)
+
+    manifest = inspect_archive(archive)
+    if manifest.get("kind") not in ("javis.voice_profile", "unknown"):
+        raise ValueError(
+            f"archive is not a Javis voice profile (kind={manifest.get('kind')!r})"
+        )
+
+    base_name = _sanitize_name(new_name or manifest.get("name") or archive.stem)
+    original_name = manifest.get("name") or base_name
+
+    voices_root.mkdir(parents=True, exist_ok=True)
+    dest_dir = voices_root / base_name
+    if dest_dir.exists() and not overwrite:
+        i = 2
+        while (voices_root / f"{base_name}_{i}").exists():
+            i += 1
+        dest_dir = voices_root / f"{base_name}_{i}"
+    elif dest_dir.exists() and overwrite:
+        shutil.rmtree(dest_dir)
+
+    dest_dir.mkdir(parents=True, exist_ok=False)
+
+    files = 0
+    total = 0
+    try:
+        with zipfile.ZipFile(archive, "r") as zf:
+            # Pre-scan: refuse pathological archives.
+            scan_total = sum(max(0, zi.file_size) for zi in zf.infolist())
+            if scan_total > _MAX_TOTAL_BYTES:
+                raise ValueError(
+                    f"archive uncompressed size {scan_total} exceeds limit "
+                    f"{_MAX_TOTAL_BYTES}"
+                )
+            for zi in zf.infolist():
+                if zi.filename == MANIFEST_NAME:
+                    continue
+                target = _safe_extract_target(dest_dir, zi.filename)
+                if target is None:
+                    log.warning(f"skipping unsafe entry: {zi.filename!r}")
+                    continue
+                if zi.is_dir():
+                    target.mkdir(parents=True, exist_ok=True)
+                    continue
+                target.parent.mkdir(parents=True, exist_ok=True)
+                with zf.open(zi, "r") as src, open(target, "wb") as dst:
+                    shutil.copyfileobj(src, dst)
+                files += 1
+                total += target.stat().st_size
+
+        # Ensure a valid profile.json with the (possibly renamed) profile name.
+        meta = dest_dir / "profile.json"
+        if meta.exists():
+            try:
+                data = json.loads(meta.read_text(encoding="utf-8"))
+            except json.JSONDecodeError:
+                data = {}
+        else:
+            data = {}
+        data.setdefault("language", manifest.get("language", "en"))
+        data.setdefault("speed", manifest.get("speed", 1.0))
+        data.setdefault("temperature", manifest.get("temperature", 0.7))
+        data["name"] = dest_dir.name
+        meta.write_text(json.dumps(data, indent=2), encoding="utf-8")
+        # Ensure refs/ exists (loader expects it).
+        (dest_dir / "refs").mkdir(exist_ok=True)
+    except Exception:
+        # Roll back partial extraction.
+        shutil.rmtree(dest_dir, ignore_errors=True)
+        raise
+
+    log.info(f"imported voice -> {dest_dir} ({files} files, {total} B)")
+    return ImportResult(
+        path=dest_dir,
+        profile_name=dest_dir.name,
+        files=files,
+        bytes=total,
+        renamed_from=original_name if original_name != dest_dir.name else None,
+    )
diff --git a/src/javis/voices/manager.py b/src/javis/voices/manager.py
new file mode 100644
index 0000000..083d0e5
--- /dev/null
+++ b/src/javis/voices/manager.py
@@ -0,0 +1,85 @@
+"""Voice profile manager."""
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from ..core.config import Config
+from ..core.logger import get_logger
+from . import io as voice_io
+from .profile import VoiceProfile
+
+log = get_logger("voices")
+
+
+class VoiceManager:
+    def __init__(self, config: Config):
+        self.config = config
+        self.root = config.paths.voices_dir
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def list(self) -> list[VoiceProfile]:
+        return [VoiceProfile.from_dir(p) for p in sorted(self.root.iterdir()) if p.is_dir()]
+
+    def get(self, name: str) -> VoiceProfile:
+        d = self.root / name
+        if not d.exists():
+            raise FileNotFoundError(f"Voice profile not found: {name}")
+        return VoiceProfile.from_dir(d)
+
+    def create(self, name: str, language: str = "en") -> VoiceProfile:
+        d = self.root / name
+        if d.exists():
+            raise FileExistsError(f"Voice profile already exists: {name}")
+        prof = VoiceProfile(name=name, language=language, dir=d)
+        prof.save()
+        log.info(f"created voice profile '{name}'")
+        return prof
+
+    def delete(self, name: str) -> None:
+        d = self.root / name
+        if d.exists():
+            shutil.rmtree(d)
+            log.info(f"deleted voice profile '{name}'")
+
+    def add_reference_clip(self, name: str, source: Path) -> Path:
+        prof_dir = self.root / name
+        refs = prof_dir / "refs"
+        refs.mkdir(parents=True, exist_ok=True)
+        dest = refs / source.name
+        shutil.copy(source, dest)
+        log.info(f"added reference '{source.name}' to '{name}'")
+        return dest
+
+    def get_or_create_default(self, name: str) -> VoiceProfile:
+        d = self.root / name
+        if d.exists():
+            return VoiceProfile.from_dir(d)
+        log.warning(
+            f"voice profile '{name}' missing. Creating empty profile at {d}. "
+            f"Drop reference WAVs into {d / 'refs'} before synthesizing."
+        )
+        return self.create(name)
+
+    # ---------------- import / export ----------------
+
+    def export(self, name: str, dest: Path) -> voice_io.ExportResult:
+        """Pack profile ``name`` into a ``.javisvoice`` archive at ``dest``.
+
+        ``dest`` may be a directory or a full file path.
+        """
+        return voice_io.export_profile(self.root / name, dest)
+
+    def import_archive(self, archive: Path,
+                       new_name: Optional[str] = None,
+                       overwrite: bool = False) -> voice_io.ImportResult:
+        """Import a ``.javisvoice`` archive into the voices directory."""
+        return voice_io.import_profile(
+            archive, self.root, new_name=new_name, overwrite=overwrite,
+        )
+
+    @staticmethod
+    def inspect_archive(archive: Path) -> dict:
+        """Read an archive's manifest without extracting it."""
+        return voice_io.inspect_archive(archive)
diff --git a/src/javis/voices/piper_prep.py b/src/javis/voices/piper_prep.py
new file mode 100644
index 0000000..2da59b6
--- /dev/null
+++ b/src/javis/voices/piper_prep.py
@@ -0,0 +1,178 @@
+"""Prepare training data for piper-train from the audiobook ingest output.
+
+The audiobook ingest pipeline produces:
+
+    {voice_dir}/training/wavs/{stem}.wav    — WAV segments (22050 Hz mono)
+    {voice_dir}/training/metadata.csv       — pipe-delimited: stem|text|text
+
+piper-train expects:
+
+    {input_dir}/wavs/{stem}.wav             — same WAV files
+    {input_dir}/metadata.csv               — same format
+
+This script bridges the gap by creating ``training/piper_input/`` which
+contains a ``metadata.csv`` copy and a ``wavs/`` junction/symlink pointing at
+``../wavs/``.  Audio data is not duplicated when symlinks/junctions are
+available; a full copy is only made as a last-resort fallback.
+
+CLI usage::
+
+    python -m javis.voices.piper_prep --voice  [--config config.yaml]
+
+Stdout: the absolute path of the prepared input directory.
+"""
+from __future__ import annotations
+
+import csv
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def prepare(voice_dir: Path, language: str = "de", sample_rate: int = 22050) -> tuple[Path, int]:
+    """Create or refresh ``training/piper_input/`` under *voice_dir*.
+
+    Returns ``(piper_input_dir, n_rows)`` where *piper_input_dir* is the
+    prepared input directory ready to be passed to
+    ``piper_train.preprocess --input-dir`` and *n_rows* is the number of
+    metadata rows.
+
+    Raises ``FileNotFoundError`` if the ingest output is missing.
+    """
+    training_dir = voice_dir / "training"
+    wavs_dir = training_dir / "wavs"
+    metadata_csv = training_dir / "metadata.csv"
+
+    if not metadata_csv.exists():
+        raise FileNotFoundError(
+            f"No training data at {metadata_csv}. "
+            "Run audiobook ingest for this voice first."
+        )
+    if not wavs_dir.is_dir():
+        raise FileNotFoundError(
+            f"No wavs directory at {wavs_dir}. "
+            "Run audiobook ingest for this voice first."
+        )
+
+    piper_input_dir = training_dir / "piper_input"
+    piper_input_dir.mkdir(exist_ok=True)
+
+    # ── metadata.csv ─────────────────────────────────────────────────────────
+    # piper-train's LJSpeech reader expects: stem|text|normalized_text
+    # Our ingest already writes exactly that format — copy verbatim.
+    dest_meta = piper_input_dir / "metadata.csv"
+    with (
+        open(metadata_csv, encoding="utf-8", newline="") as src,
+        open(dest_meta, "w", encoding="utf-8", newline="") as dst,
+    ):
+        dst.write(src.read())
+
+    with open(dest_meta, encoding="utf-8") as f:
+        n_rows = sum(1 for _ in f)
+
+    # ── wavs/ link ───────────────────────────────────────────────────────────
+    # piper-train looks for WAV files at {input_dir}/wavs/{stem}.wav.
+    # We create a junction / symlink so no data is copied.
+    wavs_link = piper_input_dir / "wavs"
+    if wavs_link.exists() or wavs_link.is_symlink():
+        # Already set up — verify it still points at the right place.
+        try:
+            if wavs_link.resolve() != wavs_dir.resolve():
+                wavs_link.unlink(missing_ok=True)
+                if wavs_link.is_dir():
+                    wavs_link.rmdir()
+        except Exception:
+            pass
+
+    if not wavs_link.exists():
+        _create_dir_link(wavs_link, wavs_dir)
+
+    # ── dataset.json ─────────────────────────────────────────────────────────
+    # piper-train's preprocess step reads this to discover the espeak locale
+    # and sample rate.  A minimal config is sufficient.
+    import json
+
+    dataset_json = piper_input_dir / "dataset.json"
+    dataset_json.write_text(
+        json.dumps(
+            {
+                "name": voice_dir.name,
+                "language": language,
+                "quality": "high",
+                "sample_rate": sample_rate,
+                "num_symbols": 256,
+                "num_speakers": 1,
+                "espeak_voice": language,
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+
+    return piper_input_dir, n_rows
+
+
+def _create_dir_link(link: Path, target: Path) -> None:
+    """Create a directory symlink/junction from *link* → *target*.
+
+    Tries three strategies in order:
+      1. ``os.symlink`` (works on Linux, macOS, and Windows with Developer Mode)
+      2. NTFS junction via ``cmd /c mklink /J`` (no admin required on Windows)
+      3. Full directory copy as a last resort (wastes space but always works)
+    """
+    try:
+        os.symlink(target, link, target_is_directory=True)
+        return
+    except (OSError, NotImplementedError):
+        pass
+
+    if sys.platform == "win32":
+        try:
+            result = subprocess.run(
+                ["cmd", "/c", "mklink", "/J", str(link), str(target)],
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode == 0:
+                return
+        except Exception:
+            pass
+
+    # Last resort: shallow hardlink-copy (original WAVs stay in clips/).
+    import shutil
+    shutil.copytree(target, link)
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def _cli() -> int:
+    import argparse
+    from ..core.config import load_config
+    from ..core.logger import setup_logger
+    from .manager import VoiceManager
+
+    p = argparse.ArgumentParser(
+        description="Prepare piper-train input directory from audiobook ingest output."
+    )
+    p.add_argument("--voice", required=True, help="Voice profile name")
+    p.add_argument("--language", default="de", help="BCP-47 language tag for espeak (default: de)")
+    p.add_argument("--config", type=Path, default=Path("config.yaml"))
+    args = p.parse_args()
+
+    config = load_config(args.config)
+    setup_logger(config)
+
+    mgr = VoiceManager(config)
+    prof = mgr.get(args.voice)
+    assert prof.dir is not None, "Voice profile has no directory"
+
+    piper_input_dir, n_rows = prepare(prof.dir, language=args.language)
+    print(f"Prepared piper-train input: {piper_input_dir}")
+    print(f"  Utterances: {n_rows:,}")
+    print(f"  Next step:  python -m piper_train.preprocess --input-dir \"{piper_input_dir}\" ...")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(_cli())
diff --git a/src/javis/voices/piper_train.py b/src/javis/voices/piper_train.py
new file mode 100644
index 0000000..0585697
--- /dev/null
+++ b/src/javis/voices/piper_train.py
@@ -0,0 +1,529 @@
+"""Piper TTS voice training pipeline runner.
+
+Uses the ``piper.train`` module from OHF-Voice/piper1-gpl to fine-tune or
+train a VITS-based TTS voice from the audiobook ingest output.
+
+Pipeline
+--------
+Stage 1 — Train (combined phoneme preprocessing + VITS training):
+    python -m piper.train fit  [--ckpt_path ...]
+
+Stage 2 — ONNX export:
+    python -m piper.train.export_onnx --checkpoint ... --output-file ...
+
+The training module (``piper.train``) is bundled in the ``piper-tts`` package
+installed via ``pip install piper-tts[train]`` plus the OHF-Voice repo at
+``D:\\piper1-gpl`` (editable install provides the training source code and
+the pure-Python monotonic_align fallback).
+
+CLI usage::
+
+    python -m javis.voices.piper_train \\
+        --voice  \\
+        [--language de] \\
+        [--base-checkpoint /path/to/base.ckpt] \\
+        [--batch-size 32] \\        [--num-workers 4] \\        [--max-epochs 6000] \\
+        [--accelerator gpu|cpu] \\
+        [--precision 32|16|bf16] \\
+        [--config config.yaml]
+
+Prerequisites
+-------------
+* ``pip install piper-tts[train]``
+* OHF-Voice/piper1-gpl cloned at ``D:\\piper1-gpl`` and installed editable
+* NVIDIA GPU with >=6 GB VRAM recommended; CPU training supported but slow
+
+Training time (RTX 3070, ~5 000 utterances)
+--------------------------------------------
+* Training 6 000 epochs, batch=32: ~4–8 h
+* Export: <1 min
+"""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+def _run(
+    cmd: list[str],
+    *,
+    cwd: Path | None = None,
+    desc: str,
+    env: dict | None = None,
+) -> int:
+    """Run *cmd* as a subprocess, streaming output to stdout/stderr.
+
+    Returns the process exit code.  Handles Ctrl-C gracefully: forwards the
+    interrupt to the child process group and waits for it to finish before
+    returning exit code 1, suppressing the noisy BrokenPipeError tracebacks
+    that Windows multiprocessing workers emit when the pipe is torn down.
+    """
+    print(f"\n{'='*60}")
+    print(f"  {desc}")
+    print(f"  {' '.join(str(a) for a in cmd)}")
+    print(f"{'='*60}\n", flush=True)
+
+    # CREATE_NEW_PROCESS_GROUP lets us send CTRL_C_EVENT to only the child.
+    flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+    proc = subprocess.Popen(cmd, cwd=cwd, env=env, creationflags=flags)
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        print("\nInterrupt received — waiting for training process to stop…",
+              flush=True)
+        try:
+            import signal as _signal
+            proc.send_signal(_signal.CTRL_C_EVENT)  # type: ignore[attr-defined]
+        except Exception:
+            proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+        return 1
+    return proc.returncode
+
+
+def _find_best_checkpoint(search_dir: Path) -> Path | None:
+    """Return the best checkpoint under *search_dir* (recursive).
+
+    Prefers ``last.ckpt`` / ``best.ckpt`` (newest by mtime across all
+    versions).  Falls back to the ``epoch=*.ckpt`` with the *numerically*
+    highest epoch number — NOT a lexicographic sort, which would pick
+    ``version_7`` over ``version_20``.
+    """
+    for name in ("last.ckpt", "best.ckpt"):
+        candidates = sorted(
+            search_dir.rglob(name),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+        if candidates:
+            return candidates[0]
+
+    ckpts = list(search_dir.rglob("epoch=*.ckpt"))
+    if not ckpts:
+        return None
+
+    def _epoch_num(p: Path) -> int:
+        m = re.search(r"epoch=(\d+)", p.name)
+        return int(m.group(1)) if m else -1
+
+    return max(ckpts, key=_epoch_num)
+
+
+# Minimum free space required on the training drive before starting a run.
+# Each ModelCheckpoint save is ~800 MB and Lightning's atomic save writes a
+# full temp copy first (~2× headroom); 25 GB also leaves room for the
+# per-version checkpoint retention (top-5 × 2 callbacks + last ≈ 8.75 GB).
+MIN_FREE_BYTES = 25 * 1024**3
+
+
+def _prune_lightning_versions(
+    lightning_dir: Path,
+    *,
+    keep_ckpt: Path | None = None,
+    keep_newest: int = 2,
+) -> None:
+    """Delete old ``lightning_logs/version_*`` folders to cap disk usage.
+
+    Every ``piper.train fit`` invocation creates a NEW ``version_N`` folder
+    (Lightning behaviour, even when resuming via ``--ckpt_path``) and piper's
+    checkpoint callbacks keep up to 10 epoch checkpoints + ``last.ckpt``
+    (~8.75 GB) per completed version.  Without pruning, each training run
+    adds several GB and eventually fills the drive mid-save (Errno 28).
+
+    Kept: the folder containing *keep_ckpt* (the resume source — must never
+    be deleted), the *keep_newest* newest folders by mtime (covers the run
+    about to start and a recently interrupted one), and empty folders.
+    Everything else is removed.
+    """
+    logs_dir = lightning_dir / "lightning_logs"
+    if not logs_dir.is_dir():
+        return
+
+    versions = [
+        p for p in logs_dir.iterdir()
+        if p.is_dir() and re.fullmatch(r"version_\d+", p.name)
+    ]
+    if len(versions) <= keep_newest + 1:
+        return  # nothing to prune
+
+    def _version_num(p: Path) -> int:
+        return int(p.name.rsplit("_", 1)[1])
+
+    versions.sort(key=_version_num)
+
+    keep: set[Path] = set()
+    if keep_ckpt is not None:
+        try:
+            # Checkpoints live at varying depths inside a version folder
+            # (e.g. version_N/checkpoints/epoch=*.ckpt or version_N/last.ckpt),
+            # so match against all ancestors rather than a fixed parent.
+            resolved = keep_ckpt.resolve()
+            for v in versions:
+                if v.resolve() in resolved.parents:
+                    keep.add(v)
+        except OSError:
+            pass
+    keep.update(sorted(versions, key=lambda p: p.stat().st_mtime,
+                       reverse=True)[:keep_newest])
+
+    pruned_gb = 0.0
+    for v in versions:
+        if v in keep:
+            continue
+        if any(v.iterdir()):
+            try:
+                size = sum(f.stat().st_size for f in v.rglob("*") if f.is_file())
+            except OSError:
+                size = 0
+            shutil.rmtree(v, ignore_errors=True)
+            if v.exists():
+                print(f"WARNING: could not fully remove {v}", file=sys.stderr)
+                continue
+            pruned_gb += size / 1024**3
+            print(f"  Pruned {v.name} ({size / 1024**3:.2f} GB)")
+
+    if pruned_gb:
+        print(f"  Pruned {pruned_gb:.2f} GB total from {logs_dir}")
+
+
+# ── Main pipeline ─────────────────────────────────────────────────────────────
+
+def run_pipeline(
+    voice_dir: Path,
+    *,
+    language: str = "de",
+    base_checkpoint: Path | None = None,
+    batch_size: int = 32,
+    num_workers: int = 2,
+    max_epochs: int = 6000,
+    precision: str = "16-mixed",
+    accelerator: str = "gpu",
+    validation_split: float = 0.01,
+    num_test_examples: int = 10,
+    sample_rate: int = 22050,
+    prefer_latest_checkpoint: bool = False,
+) -> int:
+    """Run the full piper training pipeline and return the exit code.
+
+    Reads training data from ``{voice_dir}/training/`` (produced by the
+    audiobook ingest pipeline) and writes the finished model to::
+
+        {voice_dir}/piper_model.onnx
+        {voice_dir}/piper_model.onnx.json   (voice config written during training)
+
+    If *base_checkpoint* is not given the script tries to auto-detect one in
+    ``{voice_dir}/training/piper_base/`` or resumes from the last checkpoint
+    in ``{voice_dir}/training/piper_lightning/``.  Training starts from
+    scratch if neither is found.
+
+    With *prefer_latest_checkpoint* (used by the overnight scheduler) the most
+    recent ``piper_lightning/`` checkpoint is chosen first so the run always
+    resumes from the latest progress; ``piper_base/`` is only a fallback when
+    no Lightning checkpoint exists yet.
+    """
+    training_dir = voice_dir / "training"
+    metadata_csv = training_dir / "metadata.csv"
+    wavs_dir = training_dir / "wavs"
+
+    # ── Validate ingest output ────────────────────────────────────────────────
+    if not metadata_csv.exists():
+        print(
+            f"ERROR: {metadata_csv} not found.\n"
+            "Run audiobook ingest for this voice first.",
+            file=sys.stderr,
+        )
+        return 1
+    if not wavs_dir.is_dir():
+        print(
+            f"ERROR: {wavs_dir} not found.\n"
+            "Run audiobook ingest for this voice first.",
+            file=sys.stderr,
+        )
+        return 1
+
+    with open(metadata_csv, encoding="utf-8") as f:
+        n_rows = sum(1 for _ in f)
+    print(f"Training data: {metadata_csv}")
+    print(f"  {n_rows:,} utterances in metadata.csv")
+
+    # Directories used by piper.train
+    cache_dir = training_dir / "piper_cache"
+    config_json = training_dir / "piper_config.json"
+    lightning_dir = training_dir / "piper_lightning"
+    lightning_dir.mkdir(parents=True, exist_ok=True)
+
+    # ── Preflight: disk space ─────────────────────────────────────────────
+    # Lightning's atomic checkpoint save needs ~2× the checkpoint size
+    # (~800 MB) as headroom; running out mid-save crashes the whole run
+    # (OSError: [Errno 28] No space left on device) and leaves a partial
+    # temp file behind.  Fail fast with a clear message instead.
+    free = shutil.disk_usage(lightning_dir).free
+    if free < MIN_FREE_BYTES:
+        print(
+            f"ERROR: insufficient disk space for training: "
+            f"{free / 1024**3:.1f} GB free on "
+            f"'{lightning_dir.drive or lightning_dir}', "
+            f"need ≥ {MIN_FREE_BYTES / 1024**3:.0f} GB. "
+            f"Free up space or move the voice's training directory to a "
+            f"larger drive, then retry.",
+            file=sys.stderr,
+        )
+        return 1
+
+    # ── Auto-detect base checkpoint ───────────────────────────────────────────
+    if base_checkpoint is None:
+        base_dir = training_dir / "piper_base"
+        if prefer_latest_checkpoint:
+            # Scheduler path: always resume from the newest Lightning
+            # checkpoint so each night picks up where the last run stopped.
+            base_checkpoint = _find_best_checkpoint(lightning_dir)
+            if base_checkpoint is None and base_dir.is_dir():
+                base_checkpoint = _find_best_checkpoint(base_dir)
+        else:
+            if base_dir.is_dir():
+                base_checkpoint = _find_best_checkpoint(base_dir)
+            if base_checkpoint is None:
+                base_checkpoint = _find_best_checkpoint(lightning_dir)
+
+    if base_checkpoint is not None:
+        print(f"Starting from checkpoint: {base_checkpoint}")
+    else:
+        print("No base checkpoint found — training from scratch.")
+        print(
+            "Tip: place a piper checkpoint (.ckpt) in\n"
+            f"  {training_dir / 'piper_base'}\n"
+            "to fine-tune from a pre-trained voice (much faster convergence).\n"
+            "Pre-trained checkpoints: https://huggingface.co/datasets/rhasspy/piper-checkpoints"
+        )
+
+    # ── Prune old Lightning versions (caps piper_lightning disk usage) ──
+    # After auto-detect so the version holding the resume checkpoint is
+    # protected from deletion.  Piper keeps ~8.75 GB per version (top-5 × 2
+    # callbacks + last), and every fit creates a new version_N even when
+    # resuming — without pruning this grows unbounded until the disk is full.
+    _prune_lightning_versions(lightning_dir, keep_ckpt=base_checkpoint)
+
+    # ── Stage 1: train ────────────────────────────────────────────────────────
+    train_cmd: list[str] = [
+        sys.executable, "-m", "piper.train", "fit",
+        "--data.voice_name", voice_dir.name,
+        "--data.csv_path", str(metadata_csv),
+        "--data.audio_dir", str(wavs_dir),
+        "--data.cache_dir", str(cache_dir),
+        "--data.config_path", str(config_json),
+        "--data.espeak_voice", language,
+        "--data.batch_size", str(batch_size),
+        "--data.num_workers", str(num_workers),
+        "--data.validation_split", str(validation_split),
+        "--data.num_test_examples", str(num_test_examples),
+        "--model.sample_rate", str(sample_rate),
+        "--trainer.max_epochs", str(max_epochs),
+        "--trainer.accelerator", accelerator,
+        "--trainer.devices", "1",
+        "--trainer.precision", precision,
+        "--trainer.default_root_dir", str(lightning_dir),
+    ]
+
+    if base_checkpoint is not None:
+        train_cmd += ["--ckpt_path", str(base_checkpoint)]
+
+    # Build subprocess environment: inherit everything, then overlay overrides.
+    # torch.compile is disabled: the only available backend on Windows without
+    # MSVC is aot_eager, which provides zero GPU kernel fusion (that requires
+    # inductor/Triton).  aot_eager still builds a per-shape graph cache — with
+    # variable-length phoneme/audio sequences this cache grows unboundedly,
+    # filling VRAM after ~60 iterations and collapsing throughput.
+    # Pure eager mode is faster and holds VRAM steady.
+    # PYTORCH_NO_CUDA_GRAPH_CAPTURE: prevents CUDA graph capture from holding
+    # the GPU long enough to trigger Windows TDR (driver reset / PC crash).
+    # PYTORCH_CUDA_ALLOC_CONF: the epoch-1542 OOM failed on a tiny 58 MiB
+    # allocation while 549 MiB sat in reserved-but-unallocated segments
+    # (allocator fragmentation).  expandable_segments lets the caching
+    # allocator grow segments in place instead of needing a contiguous block.
+    train_env = {**os.environ,
+                 "PYTORCH_NO_CUDA_GRAPH_CAPTURE": "1",
+                 "TORCHDYNAMO_DISABLE": "1",
+                 "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"}
+
+    rc = _run(train_cmd, env=train_env, desc="Stage 1/2 — Phoneme preprocessing + VITS training")
+    if rc != 0:
+        print(f"\nERROR: training failed (exit {rc}).", file=sys.stderr)
+        return rc
+
+    # ── Stage 2: export ONNX ─────────────────────────────────────────────────
+    best_ckpt = _find_best_checkpoint(lightning_dir)
+    if best_ckpt is None:
+        print(
+            "ERROR: no checkpoint found after training.\n"
+            f"Expected checkpoints under: {lightning_dir}",
+            file=sys.stderr,
+        )
+        return 1
+
+    onnx_out = voice_dir / "piper_model.onnx"
+    rc = _run(
+        [
+            sys.executable, "-m", "piper.train.export_onnx",
+            "--checkpoint", str(best_ckpt),
+            "--output-file", str(onnx_out),
+        ],
+        desc="Stage 2/2 — ONNX export",
+    )
+    if rc != 0:
+        print(f"\nERROR: ONNX export failed (exit {rc}).", file=sys.stderr)
+        return rc
+
+    print(f"\nDone! Model saved to: {onnx_out}")
+    if config_json.exists():
+        print(f"         Config at:   {config_json}")
+        print(f"\n  Rename these two files to use as a standard Piper voice:")
+        print(f"    {onnx_out.name}  →  -{voice_dir.name}-medium.onnx")
+        print(f"    {config_json.name}  →  -{voice_dir.name}-medium.onnx.json")
+    print("\nTo use this voice in Javis, set in config.yaml:")
+    print("  tts:")
+    print("    backend: piper")
+    return 0
+
+
+def export_snapshot(voice_dir: Path, *, output: Path | None = None) -> int:
+    """Export the best available training checkpoint to ONNX right now.
+
+    Does not start or resume training — only runs the ONNX export step on
+    whatever checkpoint exists.  Useful for previewing a voice mid-training.
+
+    Args:
+        voice_dir: The voice profile directory (contains ``training/``).
+        output:    Destination .onnx file.  Defaults to
+                   ``{voice_dir}/piper_model.onnx``.
+
+    Returns the subprocess exit code (0 = success).
+    """
+    lightning_dir = voice_dir / "training" / "piper_lightning"
+    ckpt = _find_best_checkpoint(lightning_dir)
+    if ckpt is None:
+        print(
+            f"ERROR: no checkpoint found under {lightning_dir}\n"
+            "Start training first with: python -m javis.voices.piper_train --voice ",
+            file=sys.stderr,
+        )
+        return 1
+
+    onnx_out = output or (voice_dir / "piper_model.onnx")
+    print(f"Exporting checkpoint: {ckpt}")
+    print(f"Output: {onnx_out}")
+
+    rc = _run(
+        [
+            sys.executable, "-m", "piper.train.export_onnx",
+            "--checkpoint", str(ckpt),
+            "--output-file", str(onnx_out),
+        ],
+        desc="Snapshot export — ONNX",
+    )
+    if rc != 0:
+        print(f"\nERROR: ONNX export failed (exit {rc}).", file=sys.stderr)
+        return rc
+
+    config_json = voice_dir / "training" / "piper_config.json"
+    # Copy config next to the onnx so piper can find it.
+    if config_json.exists():
+        import shutil as _shutil
+        dest_cfg = onnx_out.with_suffix(".onnx.json")
+        _shutil.copy2(config_json, dest_cfg)
+        print(f"Config copied to: {dest_cfg}")
+
+    print(f"\nSnapshot ready: {onnx_out}")
+    print("(From checkpoint:", ckpt.name, ")")
+    return 0
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def _cli() -> int:
+    import argparse
+    from ..core.config import find_project_root, load_config
+    from ..core.logger import setup_logger
+    from .manager import VoiceManager
+
+    _project_root = find_project_root()
+
+    p = argparse.ArgumentParser(
+        description="Train a custom Piper TTS voice from audiobook ingest data."
+    )
+    p.add_argument("--voice", required=True, help="Voice profile name")
+    p.add_argument("--language", default="de",
+                   help="espeak-ng voice/language code, e.g. 'de', 'en-us' (default: de)")
+    p.add_argument("--base-checkpoint", type=Path, default=None,
+                   help="Pre-trained .ckpt to fine-tune from. "
+                        "Auto-detected from training/piper_base/ when omitted.")
+    p.add_argument("--batch-size", type=int, default=32,
+                   help="Training batch size (default: 32). Batches are length-bucketed, "
+                        "so padding waste is small and VRAM stays flat instead of "
+                        "ratcheting. Measured on a 24 GB card with clips up to 15 s: "
+                        "32 runs every bucket including the longest, while 40 fits the "
+                        "average bucket but runs out of memory on the longest ones. "
+                        "Lower this if you still hit a CUDA out-of-memory error, or if "
+                        "your clips are longer than ~15 s.")
+    p.add_argument("--num-workers", type=int, default=2,
+                   help="Dataloader worker processes (default: 2). Each worker is a "
+                        "full spawned Windows process that re-imports torch/CUDA/"
+                        "lightning/librosa (~2 GB apiece) regardless of --batch-size — "
+                        "raise with caution, it costs system RAM, not VRAM.")
+    p.add_argument("--max-epochs", type=int, default=6000)
+    p.add_argument("--accelerator", default="gpu",
+                   choices=["gpu", "cpu"],
+                   help="PyTorch Lightning accelerator (default: gpu)")
+    p.add_argument("--precision", default="16-mixed",
+                   choices=["32", "16-mixed", "bf16-mixed"],
+                   help="Training precision (default: 16-mixed; use 32 for debugging)")
+    p.add_argument("--config", type=Path, default=_project_root / "config.yaml")
+    p.add_argument(
+        "--export-snapshot", action="store_true",
+        help="Export the best existing checkpoint to ONNX now (no training). "
+             "Writes {voice_dir}/piper_model.onnx and exits.",
+    )
+    p.add_argument(
+        "--snapshot-output", type=Path, default=None,
+        help="Custom .onnx output path for --export-snapshot (optional).",
+    )
+    args = p.parse_args()
+
+    config = load_config(args.config)
+    setup_logger(config)
+
+    mgr = VoiceManager(config)
+    prof = mgr.get(args.voice)
+    assert prof.dir is not None, "Voice profile has no directory"
+
+    if args.export_snapshot:
+        return export_snapshot(prof.dir, output=args.snapshot_output)
+
+    return run_pipeline(
+        prof.dir,
+        language=args.language,
+        base_checkpoint=args.base_checkpoint,
+        batch_size=args.batch_size,
+        num_workers=args.num_workers,
+        max_epochs=args.max_epochs,
+        accelerator=args.accelerator,
+        precision=args.precision,
+    )
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(_cli())
+    except KeyboardInterrupt:
+        print("\nAborted.", flush=True)
+        sys.exit(1)
diff --git a/src/javis/voices/profile.py b/src/javis/voices/profile.py
new file mode 100644
index 0000000..cb19b64
--- /dev/null
+++ b/src/javis/voices/profile.py
@@ -0,0 +1,75 @@
+"""Voice profile model."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class VoiceProfile:
+    name: str
+    language: str = "en"
+    reference_clips: list[Path] = field(default_factory=list)
+    piper_model: Optional[Path] = None   # path to trained piper_model.onnx (or None)
+    # Per-profile synthesis override.  When None the global config value is
+    # used, allowing each voice to diverge from the system default without
+    # touching config.yaml.
+    speed: Optional[float] = None
+    dir: Optional[Path] = None  # filesystem location
+
+    def to_json(self) -> dict:
+        # Reference clips are auto-discovered from /refs/*.wav, so we
+        # don't persist their paths (keeps profiles portable across machines).
+        d: dict = {
+            "name": self.name,
+            "language": self.language,
+            "piper_model": str(self.piper_model) if self.piper_model else None,
+        }
+        if self.speed is not None:
+            d["speed"] = self.speed
+        return d
+
+    @classmethod
+    def from_dir(cls, profile_dir: Path) -> "VoiceProfile":
+        meta_file = profile_dir / "profile.json"
+        data: dict = {}
+        if meta_file.exists():
+            try:
+                data = json.loads(meta_file.read_text(encoding="utf-8"))
+                if not isinstance(data, dict):
+                    raise ValueError("profile.json is not a JSON object")
+            except (json.JSONDecodeError, ValueError, OSError) as exc:
+                from ..core.logger import get_logger
+                get_logger("voices").warning(
+                    f"corrupt profile.json in {profile_dir.name} ({exc}) — using defaults"
+                )
+                data = {}
+        refs_dir = profile_dir / "refs"
+        refs = sorted(refs_dir.glob("*.wav")) if refs_dir.exists() else []
+        # speed is an optional override; absent key → None → use global config
+        speed_raw = data.get("speed")
+        try:
+            speed = float(speed_raw) if speed_raw is not None else None
+        except (TypeError, ValueError):
+            speed = None
+        # Support both old key (finetuned_model) and new (piper_model) for migration.
+        piper_model_raw = data.get("piper_model") or data.get("finetuned_model")
+        return cls(
+            name=data.get("name", profile_dir.name),
+            language=data.get("language", "en"),
+            reference_clips=refs,
+            piper_model=Path(piper_model_raw) if piper_model_raw else None,
+            speed=speed,
+            dir=profile_dir,
+        )
+
+    def save(self) -> None:
+        if self.dir is None:
+            raise ValueError("VoiceProfile.dir is not set")
+        self.dir.mkdir(parents=True, exist_ok=True)
+        (self.dir / "refs").mkdir(exist_ok=True)
+        (self.dir / "profile.json").write_text(
+            json.dumps(self.to_json(), indent=2), encoding="utf-8"
+        )
diff --git a/src/javis/voices/training_scheduler.py b/src/javis/voices/training_scheduler.py
new file mode 100644
index 0000000..a94cb5e
--- /dev/null
+++ b/src/javis/voices/training_scheduler.py
@@ -0,0 +1,310 @@
+"""Overnight scheduled voice training.
+
+Runs :func:`javis.voices.piper_train.run_pipeline` for the *currently
+selected* voice (``tts.default_voice``) inside a configurable daily time
+window (default 03:00–06:00) so the Piper model keeps improving while you
+sleep.  Each run always resumes from the latest ``piper_lightning/``
+checkpoint, so training never throws away progress — it simply continues
+where the last run (or the interactive "Train model…" button) left off.
+
+Design notes
+------------
+* **Async task.** Started with ``asyncio.create_task(training_scheduler(config,
+  bus))`` on the GUI event loop (see ``gui/main_window.py``).  The heavy
+  training itself is offloaded to a worker thread via
+  :func:`asyncio.to_thread` so the UI never blocks.
+* **Live config.** ``config.training.*`` is re-read every loop iteration, so
+  edits made in *Settings → Voice Training* take effect on the next window
+  without a restart.
+* **Window guard.** Training will not *start* if fewer than
+  ``min_remaining_minutes`` of the window are left.  A run that has already
+  started is allowed to run to completion — PyTorch-Lightning cannot be
+  cleanly killed mid-epoch, and an unfinished run is safe: it leaves a
+  checkpoint that the next run resumes from.
+* **Concurrency guard.** A lock file (``training/.training.lock``) prevents
+  the scheduler and the interactive GUI button from training the same voice
+  at the same time (they run in separate processes).
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+
+from ..core.config import Config
+from ..core.logger import get_logger
+
+log = get_logger("voices.training")
+
+# A lock is considered "live" for this long.  Overnight runs take a few
+# hours; 48 h comfortably bounds a stuck/crashed run so it self-expires.
+_LOCK_STALE_SECONDS = 48 * 60 * 60
+
+
+# ── Pure helpers (unit-testable) ─────────────────────────────────────────────
+
+def next_window_start(now: datetime, start_hour: int) -> datetime:
+    """Return the next time today-or-tomorrow at ``start_hour:00`` (after *now*).
+
+    ``start_hour`` is a 0–23 local hour.  If the window already passed *now*,
+    the next day's window is returned.  A time that is *exactly* on the hour
+    is still "the window starting now" (``<``, not ``<=``), otherwise the
+    scheduler would skip the entire window it was about to open.
+    """
+    candidate = now.replace(hour=start_hour, minute=0, second=0, microsecond=0)
+    if candidate < now:
+        candidate += timedelta(days=1)
+    return candidate
+
+
+def next_run_time(now: datetime, start_hour: int, end_hour: int) -> datetime:
+    """Return the time the next training run should start.
+
+    If the daily window is *currently open* (it may have opened today, or
+    yesterday for a window that wraps past midnight), returns ``now`` so the
+    run starts immediately.  Otherwise returns the next window start.  This
+    means a scheduler that (re)starts mid-window trains right away instead of
+    sleeping until tomorrow and skipping the window that is already open.
+    """
+    for offset in (0, -1):
+        start = (now + timedelta(days=offset)).replace(
+            hour=start_hour, minute=0, second=0, microsecond=0)
+        end = start.replace(hour=end_hour, minute=0, second=0, microsecond=0)
+        if end <= start:  # window wraps past midnight
+            end += timedelta(days=1)
+        if start <= now < end:
+            return now
+    return next_window_start(now, start_hour)
+
+
+def window_end(now: datetime, start_hour: int, end_hour: int) -> datetime:
+    """Return the closing time of the window that contains *now* (or just passed).
+
+    Supports a window that wraps past midnight (e.g. 23 → 2).
+    """
+    # The window that is currently open (or the most recent one) opened at the
+    # most recent start_hour.  Compute its close.
+    start = now.replace(hour=start_hour, minute=0, second=0, microsecond=0)
+    # If we're before today's start, use yesterday's window (it may wrap).
+    if now < start:
+        start -= timedelta(days=1)
+    end = start.replace(hour=end_hour, minute=0, second=0, microsecond=0)
+    if end <= start:  # window wraps past midnight
+        end += timedelta(days=1)
+    return end
+
+
+# ── Cross-process lock ────────────────────────────────────────────────────────
+
+def _lock_path(voice_dir: Path) -> Path:
+    return voice_dir / "training" / ".training.lock"
+
+
+def _lock_alive(voice_dir: Path) -> bool:
+    """True if a recent lock file indicates another training is in progress."""
+    lp = _lock_path(voice_dir)
+    if not lp.exists():
+        return False
+    try:
+        data = json.loads(lp.read_text(encoding="utf-8"))
+        ts = float(data.get("started_epoch", 0.0))
+    except Exception:
+        try:
+            ts = lp.stat().st_mtime
+        except OSError:
+            return False
+    if not ts:
+        return False
+    return (time.time() - ts) < _LOCK_STALE_SECONDS
+
+
+def _acquire_lock(voice_dir: Path) -> bool:
+    """Atomically claim the training lock.  Returns False if already held."""
+    lp = _lock_path(voice_dir)
+    lp.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        # 'x' = exclusive create; fails if a live lock already exists.
+        with open(lp, "x", encoding="utf-8") as f:
+            f.write(json.dumps({"started_epoch": time.time()}))
+        return True
+    except FileExistsError:
+        return False
+    except OSError:
+        # On platforms where 'x' is flaky, fall back to the liveness check.
+        return not _lock_alive(voice_dir)
+
+
+def _release_lock(voice_dir: Path) -> None:
+    try:
+        _lock_path(voice_dir).unlink(missing_ok=True)
+    except OSError:
+        pass
+
+
+def training_in_progress(voice_dir: Path) -> bool:
+    """Public helper: is another training of this voice currently running?"""
+    return _lock_alive(voice_dir)
+
+
+# ── One run ───────────────────────────────────────────────────────────────────
+
+async def _run_once(config: Config, voice_dir: Path, bus: Optional[object] = None) -> None:
+    """Execute a single scheduled training run (threaded) with locking."""
+    from .piper_train import run_pipeline  # imported lazily (heavy deps)
+
+    t = config.training
+
+    now = datetime.now()
+    end = window_end(now, t.start_hour, t.end_hour)
+    remaining_min = (end - now).total_seconds() / 60.0
+    if remaining_min < t.min_remaining_minutes:
+        log.info(
+            f"Training window almost over ({remaining_min:.0f} min left < "
+            f"{t.min_remaining_minutes} min minimum); skipping start.")
+        return
+
+    if _lock_alive(voice_dir):
+        log.info("Another training of this voice is already running; skipping.")
+        return
+
+    if not _acquire_lock(voice_dir):
+        log.info("Could not acquire training lock (already held); skipping.")
+        return
+
+    voice = voice_dir.name
+    n_clips = 0
+    meta = voice_dir / "training" / "metadata.csv"
+    if meta.exists():
+        try:
+            n_clips = sum(1 for _ in meta.open(encoding="utf-8", errors="ignore"))
+        except OSError:
+            pass
+
+    log.info(
+        f"Scheduled training starting: voice={voice!r} window until "
+        f"{end.strftime('%H:%M')} "
+        f"({n_clips} utterances, batch={t.batch_size}, max_epochs={t.max_epochs})"
+    )
+
+    # Surface to the GUI (best-effort; bus may be None in tests / CLI).
+    if bus is not None:
+        await _safe_publish(
+            bus, "task_start",
+            {"id": "training", "name": f"Overnight voice training: {voice}"})
+
+    def _work() -> int:
+        # base_checkpoint=None + prefer_latest_checkpoint=True → resume from the
+        # newest piper_lightning checkpoint (or piper_base fallback), exactly
+        # what "start on the latest checkpoint" requires.
+        return run_pipeline(
+            voice_dir,
+            language=t.language,
+            base_checkpoint=None,
+            batch_size=t.batch_size,
+            num_workers=t.num_workers,
+            max_epochs=t.max_epochs,
+            accelerator=t.accelerator,
+            precision=t.precision,
+            prefer_latest_checkpoint=True,
+        )
+
+    rc: int
+    try:
+        rc = await asyncio.to_thread(_work)
+    except Exception as exc:  # noqa: BLE001 — training must never kill the loop
+        log.exception(f"Scheduled training crashed: {exc}")
+        rc = -1
+    finally:
+        _release_lock(voice_dir)
+
+    status = "completed" if rc == 0 else f"FAILED (exit {rc})"
+    log.info(f"Scheduled training {status}: voice={voice!r}")
+    if bus is not None:
+        await _safe_publish(
+            bus, "task_done",
+            {"id": "training", "ok": rc == 0,
+             "msg": f"Voice training {status}"})
+
+
+async def _safe_publish(bus, event: str, payload: dict) -> None:
+    try:
+        await bus.publish(event, payload)  # type: ignore[attr-defined]
+    except Exception:
+        pass
+
+
+# ── Main loop ─────────────────────────────────────────────────────────────────
+
+async def training_scheduler(config: Config, bus: Optional[object] = None) -> None:
+    """Long-lived async task: train the selected voice each night's window.
+
+    Re-reads ``config.training.*`` every iteration so Settings edits apply
+    without a restart.  Never raises out of the loop.
+    """
+    log.info(
+        f"Training scheduler started (window "
+        f"{config.training.start_hour:02d}:00–{config.training.end_hour:02d}:00, "
+        f"voice={config.tts.default_voice!r})")
+
+    while True:
+        try:
+            t = config.training
+            if not t.enabled:
+                # Poll cheaply so toggling the setting on takes effect soon.
+                await asyncio.sleep(60)
+                continue
+
+            now = datetime.now()
+            # If the window is already open (e.g. Javis started mid-window),
+            # run immediately instead of sleeping until tomorrow.
+            start = next_run_time(now, t.start_hour, t.end_hour)
+            wait_s = (start - now).total_seconds()
+
+            voice = (config.tts.default_voice or "").strip()
+            if not voice:
+                log.info("No default voice set; nothing to train.")
+                # Outside the window: sleep until it opens.  Inside it:
+                # poll every minute so a newly set voice is picked up.
+                await asyncio.sleep(max(wait_s, 60.0))
+                continue
+
+            voice_dir = config.paths.voices_dir / voice
+            if not voice_dir.exists():
+                log.warning(f"Voice dir {voice_dir!r} does not exist.")
+                await asyncio.sleep(max(wait_s, 60.0))
+                continue
+            if not (voice_dir / "training" / "metadata.csv").exists():
+                log.info(
+                    f"No training data for {voice!r} yet "
+                    f"(run audiobook ingest first).")
+                await asyncio.sleep(max(wait_s, 60.0))
+                continue
+
+            log.info(
+                    f"Next training window in {wait_s / 3600.0:.1f} h "
+                    f"({start.strftime('%a %H:%M')}). Sleeping.")
+            await asyncio.sleep(wait_s)
+
+            await _run_once(config, voice_dir, bus)
+
+            # One run per night: after a run finishes (or fails fast, e.g.
+            # the disk-space preflight), wait for the next day's window
+            # instead of re-running immediately in a tight loop.
+            now = datetime.now()
+            wait_s = (next_window_start(now, t.start_hour) - now).total_seconds()
+            log.info(
+                f"Run finished; next training window in "
+                f"{wait_s / 3600.0:.1f} h "
+                f"({next_window_start(now, t.start_hour).strftime('%a %H:%M')}). "
+                f"Sleeping.")
+            await asyncio.sleep(max(wait_s, 0.0))
+
+        except asyncio.CancelledError:
+            log.info("Training scheduler cancelled.")
+            raise
+        except Exception as exc:  # noqa: BLE001 — keep the loop alive
+            log.exception(f"Training scheduler iteration failed: {exc}")
+            await asyncio.sleep(60)
diff --git a/tests/_retheme_check.py b/tests/_retheme_check.py
new file mode 100644
index 0000000..5804ede
--- /dev/null
+++ b/tests/_retheme_check.py
@@ -0,0 +1,118 @@
+"""Offscreen functional check for the GUI retheme registry.
+
+Single thread (Qt widgets must be driven on their creating thread):
+- Patch the modal startup dialog to return the config immediately.
+- Patch ``qasync.QEventLoop.run_forever`` to grab the top-level window and raise
+  a sentinel BEFORE the GUI loop actually runs.
+- Call ``run_gui``, catch the sentinel (window is now fully constructed), then
+  read a registered themed widget's style in dark, trigger the REAL light/dark
+  toggle (``_theme_act`` -> ``_on_theme_toggle`` -> ``_apply_theme`` ->
+  ``retheme``), and assert the style changed and round-trips.
+
+Run:  QT_QPA_PLATFORM=offscreen .\\.venv\\Scripts\\python.exe tests\\_retheme_check.py
+"""
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
+
+from PySide6 import QtWidgets  # noqa: E402
+import qasync  # noqa: E402
+
+from javis.core.config import load_config  # noqa: E402
+import javis.gui.main_window as mw  # noqa: E402
+
+
+class _Stop(Exception):
+    pass
+
+
+STATE: dict = {"win": None}
+
+
+def _probe(win) -> None:
+    """Drive the window: capture dark, toggle to light, assert it changed."""
+    if not getattr(win, "_themed", None):
+        raise AssertionError("retheme registry (_themed) is empty!")
+
+    sample = None
+    for w, factory in win._themed:
+        try:
+            factory()
+        except Exception:
+            continue
+        sample = (w, factory)
+        break
+    if sample is None:
+        w, factory = win._themed[0]
+        sample = (w, factory)
+
+    w, factory = sample
+    app = QtWidgets.QApplication.instance()
+    app.processEvents()
+    dark_style = w.styleSheet()
+    print(f"[probe] themed widget: {type(w).__name__}")
+    print(f"[probe] dark  style = {dark_style!r}")
+
+    act = win._theme_act
+    act.setChecked(True)   # -> light
+    app.processEvents()
+    light_style = w.styleSheet()
+    print(f"[probe] light style = {light_style!r}")
+
+    assert light_style != dark_style, (
+        "BUG: toggling theme did not change the registered widget's stylesheet. "
+        f"dark={dark_style!r} light={light_style!r}"
+    )
+
+    act.setChecked(False)  # -> dark
+    app.processEvents()
+    back_dark = w.styleSheet()
+    print(f"[probe] dark  (again) = {back_dark!r}")
+    assert back_dark == dark_style, (
+        f"BUG: toggling back did not restore the dark style. "
+        f"{back_dark!r} != {dark_style!r}"
+    )
+
+    print("\n[probe] OK — retheme registry is theme-reactive (dark->light->dark round-trip).")
+
+
+def main() -> int:
+    app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([])
+
+    config = load_config(Path("config.yaml"))
+    mw.show_startup_dialog = lambda cfg: cfg  # type: ignore[assignment]
+
+    def _run_forever(self):
+        tops = [w for w in QtWidgets.QApplication.topLevelWidgets()
+                if isinstance(w, QtWidgets.QMainWindow) and getattr(w, "retheme", None)]
+        if tops:
+            STATE["win"] = tops[-1]
+        raise _Stop()
+
+    qasync.QEventLoop.run_forever = _run_forever  # type: ignore[assignment]
+
+    try:
+        mw.run_gui(config)
+    except _Stop:
+        pass
+
+    if STATE["win"] is None:
+        print("[FAIL] could not capture the main window")
+        return 1
+
+    try:
+        _probe(STATE["win"])
+    except Exception as e:  # noqa: BLE001
+        print(f"[FAIL] {e!r}")
+        return 1
+
+    print("\nRESULT: PASS")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/test_fixes.py b/tests/test_fixes.py
new file mode 100644
index 0000000..506c4ef
--- /dev/null
+++ b/tests/test_fixes.py
@@ -0,0 +1,404 @@
+"""Regression tests for the correctness fixes applied to Javis.
+
+These target concrete, previously-broken behaviours and use only
+dependency-light modules (no torch / audio / MCP) so they run fast.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from javis.core.config import Config, PathsConfig, find_project_root
+from javis.llm.conversation import Conversation, count_tokens
+from javis.voices.profile import VoiceProfile
+
+
+class FindProjectRootTests(unittest.TestCase):
+    def test_finds_pyproject_marker(self) -> None:
+        root = find_project_root()
+        # The repository root holds pyproject.toml.
+        self.assertTrue((root / "pyproject.toml").is_file())
+
+    def test_falls_back_to_cwd_without_marker(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            deep = Path(td) / "a" / "b"
+            deep.mkdir(parents=True)
+            # No pyproject.toml anywhere up the temp tree → cwd fallback, no raise.
+            result = find_project_root(deep)
+            self.assertIsInstance(result, Path)
+
+
+class VoiceProfileCorruptionTests(unittest.TestCase):
+    def test_corrupt_profile_json_does_not_raise(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            d = Path(td)
+            (d / "profile.json").write_text("{ this is not valid json", encoding="utf-8")
+            prof = VoiceProfile.from_dir(d)  # must not raise
+            self.assertEqual(prof.name, d.name)
+            self.assertIsNone(prof.speed)
+
+    def test_non_numeric_speed_falls_back_to_none(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            d = Path(td)
+            (d / "profile.json").write_text(
+                json.dumps({"name": "x", "speed": "fast"}), encoding="utf-8"
+            )
+            prof = VoiceProfile.from_dir(d)
+            self.assertIsNone(prof.speed)
+
+    def test_valid_speed_is_parsed(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            d = Path(td)
+            (d / "profile.json").write_text(
+                json.dumps({"name": "x", "speed": 1.25}), encoding="utf-8"
+            )
+            prof = VoiceProfile.from_dir(d)
+            self.assertEqual(prof.speed, 1.25)
+
+
+class ConversationTests(unittest.TestCase):
+    def test_history_property_is_a_readonly_snapshot(self) -> None:
+        conv = Conversation(system_prompt="sys", max_turns=20)
+        conv.add_user("hello")
+        snap = conv.history
+        snap.append({"role": "user", "content": "mutated"})
+        # Mutating the snapshot must not affect the real history.
+        self.assertEqual(len(conv.history), 1)
+
+    def test_trim_keeps_tool_pairs_and_bounds_size(self) -> None:
+        conv = Conversation(system_prompt="sys", max_turns=2, max_chars=200)
+        for i in range(10):
+            conv.add_user("u" * 50)
+            conv.add_assistant("a" * 50)
+        total = sum(len(m.get("content") or "") for m in conv.history)
+        self.assertLessEqual(total, 200)
+        # History must never start with an orphaned tool/assistant message.
+        if conv.history:
+            self.assertEqual(conv.history[0]["role"], "user")
+
+    def test_count_tokens_fallback_is_nonzero_for_text(self) -> None:
+        self.assertEqual(count_tokens(""), 0)
+        self.assertGreater(count_tokens("hello world this is text"), 0)
+
+
+class QuickEngineTests(unittest.TestCase):
+    def test_load_skips_malformed_command_but_keeps_valid(self) -> None:
+        from javis.agent.quick_engine import QuickEngine
+
+        with tempfile.TemporaryDirectory() as td:
+            path = Path(td) / "quick_commands.json"
+            path.write_text(json.dumps({
+                "version": 1,
+                "commands": {
+                    "good": {
+                        "id": "good", "trigger": "lights on", "match_mode": "exact",
+                        "actions": [{"tool": "homey__x", "args": {}}],
+                    },
+                    "bad": {
+                        "id": "bad", "trigger": "broken", "match_mode": "exact",
+                        "actions": [{"not_a_valid_field": 1}],
+                    },
+                },
+            }), encoding="utf-8")
+            eng = QuickEngine(path)
+            eng.load()  # must not raise
+            ids = {c.id for c in eng.list_all()}
+            self.assertIn("good", ids)
+            self.assertNotIn("bad", ids)
+
+
+class DeviceStoreDebounceTests(unittest.TestCase):
+    def test_touch_last_seen_debounces_disk_writes(self) -> None:
+        from javis.api.devices import SlaveDeviceStore
+
+        with tempfile.TemporaryDirectory() as td:
+            store = SlaveDeviceStore(Path(td) / "slaves.json")
+            store.register("dev1", "Device One")
+            with patch.object(store, "_save", wraps=store._save) as spy:
+                # Same IP, rapid calls → at most one flush within the interval.
+                for _ in range(5):
+                    store.touch_last_seen("dev1", ip_address="10.0.0.1")
+                self.assertLessEqual(spy.call_count, 1)
+
+    def test_touch_last_seen_flushes_on_ip_change(self) -> None:
+        from javis.api.devices import SlaveDeviceStore
+
+        with tempfile.TemporaryDirectory() as td:
+            store = SlaveDeviceStore(Path(td) / "slaves.json")
+            store.register("dev1", "Device One")
+            store.touch_last_seen("dev1", ip_address="10.0.0.1")
+            with patch.object(store, "_save", wraps=store._save) as spy:
+                store.touch_last_seen("dev1", ip_address="10.0.0.2")  # IP changed
+                self.assertEqual(spy.call_count, 1)
+
+
+class TimerPersistenceTests(unittest.IsolatedAsyncioTestCase):
+    async def asyncNoop(self, timer_id: str, label: str) -> None:
+        return None
+
+    async def test_set_persists_and_cancel_updates_file(self) -> None:
+        from javis.core.timer import TimerService
+
+        with tempfile.TemporaryDirectory() as td:
+            path = Path(td) / "timers.json"
+            svc = TimerService(self.asyncNoop, persist_path=path)
+            tid = svc.set_timer("test", 3600)
+            try:
+                self.assertTrue(path.exists())
+                data = json.loads(path.read_text(encoding="utf-8"))
+                self.assertEqual(len(data["timers"]), 1)
+                self.assertTrue(svc.cancel_timer(tid))
+                data = json.loads(path.read_text(encoding="utf-8"))
+                self.assertEqual(len(data["timers"]), 0)
+            finally:
+                svc.stop()
+
+    async def test_restore_reschedules_pending_and_drops_expired(self) -> None:
+        import time as _time
+        from javis.core.timer import TimerService
+
+        with tempfile.TemporaryDirectory() as td:
+            path = Path(td) / "timers.json"
+            now = _time.time()
+            path.write_text(json.dumps({
+                "version": 1,
+                "timers": [
+                    {"id": "pending", "label": "future", "duration_s": 3600,
+                     "fire_at": now + 3600},
+                    {"id": "expired", "label": "past", "duration_s": 60,
+                     "fire_at": now - 10},
+                ],
+            }), encoding="utf-8")
+            svc = TimerService(self.asyncNoop, persist_path=path)
+            await svc.restore()
+            try:
+                active = svc.list_timers()
+                self.assertEqual(len(active), 1)
+                self.assertEqual(active[0]["id"], "pending")
+            finally:
+                svc.stop()
+
+
+class FinetuneSignatureTests(unittest.IsolatedAsyncioTestCase):
+    async def test_run_pipeline_receives_profile_dir_not_config(self) -> None:
+        from javis.voices.manager import VoiceManager
+        import javis.voices.piper_train as pt
+
+        with tempfile.TemporaryDirectory() as td:
+            config = Config()
+            config.paths = PathsConfig(base_dir=Path(td))
+            config.paths.ensure()
+            mgr = VoiceManager(config)
+            prof = mgr.create("tv")
+            assert prof.dir is not None
+            (prof.dir / "training").mkdir(parents=True, exist_ok=True)
+            (prof.dir / "training" / "metadata.csv").write_text(
+                "clip_0|hallo|hallo\n", encoding="utf-8")
+
+            captured: dict = {}
+
+            def fake_run_pipeline(voice_dir, **kwargs):
+                captured["voice_dir"] = voice_dir
+                captured["kwargs"] = kwargs
+                return 0
+
+            from javis.voices import finetune
+            with patch.object(pt, "run_pipeline", fake_run_pipeline):
+                rc = await finetune.finetune_voice(config, "tv", language="de")
+
+            self.assertEqual(rc, 0)
+            self.assertEqual(captured["voice_dir"], prof.dir)
+            # The old bug passed the Config as voice_dir and voice_name as language.
+            self.assertNotIsInstance(captured["voice_dir"], Config)
+            self.assertEqual(captured["kwargs"].get("language"), "de")
+
+
+# ── wake-word barge-in (interrupt TTS playback with the wake phrase) ──────
+
+
+class BargeInConfigTests(unittest.TestCase):
+    """The new WakeWordConfig.barge_in_enabled flag defaults to True."""
+
+    def test_barge_in_enabled_defaults_true(self) -> None:
+        from javis.core.config import WakeWordConfig
+
+        self.assertTrue(WakeWordConfig().barge_in_enabled)
+
+
+class BargeInListenerTests(unittest.IsolatedAsyncioTestCase):
+    """_barge_in_listener: on wake-word trigger it cancels playback, resets
+    the detector, sets the barge-in flag, and publishes the event."""
+
+    async def test_trigger_cancels_playback_and_sets_flag(self) -> None:
+        from javis.agent.orchestrator import Orchestrator
+        from javis.core.state import AgentState
+
+        orch = object.__new__(Orchestrator)
+
+        class FakeWake:
+            def __init__(self) -> None:
+                self.feed_calls = 0
+                self.reset_calls = 0
+
+            def feed(self, chunk):
+                self.feed_calls += 1
+                return 0.9 if self.feed_calls == 1 else 0.1
+
+            def triggered(self, score):
+                return score >= 0.5
+
+            def reset(self):
+                self.reset_calls += 1
+
+        class FakeMic:
+            def __init__(self) -> None:
+                self.drain_calls = 0
+
+            def drain(self):
+                self.drain_calls += 1
+
+            async def chunks(self, stale_timeout=None):
+                yield b"\x00"
+
+        class FakeBus:
+            def __init__(self) -> None:
+                self.events = []
+
+            async def publish(self, name, payload=None):
+                self.events.append((name, payload))
+
+        orch.wake = FakeWake()
+        orch.mic = FakeMic()
+        orch.bus = FakeBus()
+        orch._stop = asyncio.Event()
+        orch._barge_in_requested = False
+        orch._barge_in_task = None
+
+        interrupt_calls = []
+        state_calls = []
+        with patch.object(Orchestrator, "interrupt",
+                          lambda self: interrupt_calls.append(1)), \
+             patch.object(Orchestrator, "set_state",
+                          lambda self, s: state_calls.append(s) or asyncio.sleep(0)), \
+             patch.object(Orchestrator, "_play_wake_chime",
+                          lambda self: asyncio.sleep(0)):
+            await orch._barge_in_listener()
+
+        # Trigger fired on the first chunk: listener must have exited.
+        self.assertTrue(orch._barge_in_requested)
+        self.assertEqual(len(interrupt_calls), 1)
+        self.assertEqual(state_calls, [AgentState.WAKE])
+        # reset() is called once at arming and once after the trigger.
+        self.assertEqual(orch.wake.reset_calls, 2)
+        self.assertEqual(orch.mic.drain_calls, 1)
+        self.assertIn(("barge_in", 0.9), orch.bus.events)
+        # Chime was scheduled via ensure_future — let it finish.
+        await asyncio.sleep(0)
+
+
+class BargeInRunConsumptionTests(unittest.IsolatedAsyncioTestCase):
+    """run(): when _barge_in_requested is set, the main loop must skip
+    _wait_for_wake, drain the mic tail, and start the turn with followup
+    grace instead of the default 1500 ms."""
+
+    async def test_run_consumes_barge_in_flag(self) -> None:
+        from javis.agent.orchestrator import Orchestrator
+
+        orch = object.__new__(Orchestrator)
+        orch.stt = object()
+        orch.mic = object()
+        orch.wake = object()
+        orch._stop = asyncio.Event()
+        orch._turn_lock = asyncio.Lock()
+        orch._barge_in_requested = True
+        orch._barge_in_task = None
+        orch._followup_pending = False
+        orch.config = Config()
+        orch.profiler = None
+
+        class FakeMic:
+            def __init__(self) -> None:
+                self.drain_calls = 0
+
+            def drain(self):
+                self.drain_calls += 1
+
+        orch.mic = FakeMic()
+
+        class FakeBus:
+            def __init__(self) -> None:
+                self.events = []
+
+            async def publish(self, name, payload=None):
+                self.events.append((name, payload))
+
+        orch.bus = FakeBus()
+
+        wake_calls = []
+        turn_calls = []
+
+        async def fake_handle_turn(self, initial_grace_ms=1500):
+            turn_calls.append(initial_grace_ms)
+            self._stop.set()  # end the run() loop after the first turn
+
+        with patch.object(Orchestrator, "_wait_for_wake",
+                          lambda self: wake_calls.append(1)), \
+             patch.object(Orchestrator, "_handle_turn", fake_handle_turn), \
+             patch.object(Orchestrator, "stop",
+                          lambda self: asyncio.sleep(0)), \
+             patch.object(Orchestrator, "set_state",
+                          lambda self, s: asyncio.sleep(0)):
+            await orch.run()
+
+        self.assertFalse(orch._barge_in_requested)
+        self.assertEqual(orch.mic.drain_calls, 1)
+        self.assertEqual(wake_calls, [])
+        self.assertEqual(turn_calls, [int(orch.config.wake_word.followup_grace_s * 1000)])
+        self.assertIn(("barge_in_listen", {}), orch.bus.events)
+
+
+class BargeInEmptyUtteranceTests(unittest.IsolatedAsyncioTestCase):
+    """_handle_turn: an empty capture must publish stt_done(empty) + stt_empty
+    and reset the state to IDLE (previously the GUI stayed stuck on
+    'listening')."""
+
+    async def test_empty_utterance_resets_to_idle(self) -> None:
+        from javis.agent.orchestrator import Orchestrator
+        from javis.core.state import AgentState
+
+        orch = object.__new__(Orchestrator)
+        orch.stt = object()
+        orch.mic = object()
+        orch.vad = object()
+        orch.config = Config()
+        orch._last_response_is_question = False
+        orch.profiler = None
+
+        class FakeBus:
+            def __init__(self) -> None:
+                self.events = []
+
+            async def publish(self, name, payload=None):
+                self.events.append((name, payload))
+
+        orch.bus = FakeBus()
+
+        state_calls = []
+        with patch("javis.agent.orchestrator.record_utterance",
+                   new=lambda *a, **kw: asyncio.sleep(0, result=None)), \
+             patch.object(Orchestrator, "set_state",
+                          lambda self, s: state_calls.append(s) or asyncio.sleep(0)):
+            await orch._handle_turn(initial_grace_ms=1500)
+
+        self.assertIn(("stt_done", {"text": "", "audio_s": 0.0, "latency_s": 0.0, "empty": True}),
+                      orch.bus.events)
+        self.assertIn(("stt_empty", {}), orch.bus.events)
+        self.assertEqual(state_calls, [AgentState.LISTENING, AgentState.IDLE])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_hermes_memory.py b/tests/test_hermes_memory.py
new file mode 100644
index 0000000..f839d0e
--- /dev/null
+++ b/tests/test_hermes_memory.py
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import httpx
+
+from javis.agent.middleware import MemoryMiddleware, TurnContext
+from javis.core.config import Config, PathsConfig
+from javis.llm.hermes_client import HermesClient
+from javis.llm.ollama_client import ChatResponse
+
+
+class HermesClientTests(unittest.IsolatedAsyncioTestCase):
+    async def test_chat_uses_bearer_auth_and_openai_payload(self) -> None:
+        config = Config()
+        config.memory.provider = "hermes"
+        config.memory.base_url = "http://hermes.test:30432"
+        config.memory.model = "hermes-agent"
+        config.memory.api_key_env = "TEST_HERMES_KEY"
+        captured: dict = {}
+
+        async def handler(request: httpx.Request) -> httpx.Response:
+            captured["url"] = str(request.url)
+            captured["authorization"] = request.headers.get("Authorization")
+            captured["payload"] = json.loads(request.content)
+            return httpx.Response(
+                200,
+                json={
+                    "choices": [
+                        {"message": {"role": "assistant", "content": "[]"}}
+                    ]
+                },
+            )
+
+        client = HermesClient(config)
+        await client._client.aclose()
+        client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        try:
+            with patch.dict(os.environ, {"TEST_HERMES_KEY": "test-secret"}):
+                response = await client.chat(
+                    [{"role": "user", "content": "Extract memories"}],
+                    temperature=0.0,
+                )
+        finally:
+            await client.aclose()
+
+        self.assertEqual(captured["url"], "http://hermes.test:30432/v1/chat/completions")
+        self.assertEqual(captured["authorization"], "Bearer test-secret")
+        self.assertEqual(captured["payload"]["model"], "hermes-agent")
+        self.assertFalse(captured["payload"]["stream"])
+        self.assertEqual(response.content, "[]")
+
+    async def test_list_models_is_authenticated(self) -> None:
+        config = Config()
+        config.memory.provider = "hermes"
+        config.memory.base_url = "http://hermes.test:30432"
+        config.memory.api_key_env = "TEST_HERMES_KEY"
+        authorization = ""
+
+        async def handler(request: httpx.Request) -> httpx.Response:
+            nonlocal authorization
+            authorization = request.headers.get("Authorization", "")
+            return httpx.Response(200, json={"data": [{"id": "hermes-agent"}]})
+
+        client = HermesClient(config)
+        await client._client.aclose()
+        client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        try:
+            with patch.dict(os.environ, {"TEST_HERMES_KEY": "test-secret"}):
+                models = await client.list_models()
+        finally:
+            await client.aclose()
+
+        self.assertEqual(authorization, "Bearer test-secret")
+        self.assertEqual(models, ["hermes-agent"])
+
+
+class FakeMemoryClient:
+    async def chat(self, messages: list[dict], **kwargs) -> ChatResponse:
+        return ChatResponse(content=json.dumps([
+            {
+                "key": "music_preference",
+                "value": "Florian prefers jazz music",
+                "category": "preferences",
+            }
+        ]))
+
+
+class HermesMemoryMiddlewareTests(unittest.IsolatedAsyncioTestCase):
+    async def test_short_explicit_request_is_extracted_and_persisted(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            config = Config(paths=PathsConfig(base_dir=Path(temporary_directory)))
+            config.memory.provider = "hermes"
+            config.memory.model = "hermes-agent"
+            middleware = MemoryMiddleware(FakeMemoryClient(), config)
+            context = TurnContext(turn_id="turn-1", speaker_name="Florian")
+
+            await middleware.on_finish(
+                context,
+                "Merke dir: Ich mag Jazz.",
+                "Das merke ich mir.",
+            )
+
+            self.assertEqual(len(context._deferred), 1)
+            await context._deferred[0]
+            personal_file = Path(temporary_directory) / "memory_florian.json"
+            stored = json.loads(personal_file.read_text(encoding="utf-8"))
+            self.assertEqual(
+                stored["entries"]["music_preference"]["value"],
+                "Florian prefers jazz music",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()
\ No newline at end of file
diff --git a/voice-ref-curator/VoiceRefCurator.spec b/voice-ref-curator/VoiceRefCurator.spec
new file mode 100644
index 0000000..eaa3cd8
--- /dev/null
+++ b/voice-ref-curator/VoiceRefCurator.spec
@@ -0,0 +1,44 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+a = Analysis(
+    ['src\\voice_ref_curator\\__main__.py'],
+    pathex=['src'],
+    binaries=[],
+    datas=[],
+    hiddenimports=['voice_ref_curator', 'voice_ref_curator.analyser', 'voice_ref_curator.gui'],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='VoiceRefCurator',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=False,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='VoiceRefCurator',
+)
diff --git a/voice-ref-curator/build.bat b/voice-ref-curator/build.bat
new file mode 100644
index 0000000..95fd359
--- /dev/null
+++ b/voice-ref-curator/build.bat
@@ -0,0 +1,24 @@
+@echo off
+:: Build a standalone Windows executable with PyInstaller.
+:: Output: dist\VoiceRefCurator\VoiceRefCurator.exe
+cd /d "%~dp0"
+call ..\.venv\Scripts\activate.bat
+
+pip install pyinstaller --quiet
+
+pyinstaller ^
+  --windowed ^
+  --onedir ^
+  --name VoiceRefCurator ^
+  --paths src ^
+  --hidden-import voice_ref_curator ^
+  --hidden-import voice_ref_curator.analyser ^
+  --hidden-import voice_ref_curator.gui ^
+  src\voice_ref_curator\__main__.py
+
+echo.
+echo ============================================================
+echo  Build complete.
+echo  Executable: dist\VoiceRefCurator\VoiceRefCurator.exe
+echo ============================================================
+pause
diff --git a/voice-ref-curator/pyproject.toml b/voice-ref-curator/pyproject.toml
new file mode 100644
index 0000000..2c9a14e
--- /dev/null
+++ b/voice-ref-curator/pyproject.toml
@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "voice-ref-curator"
+version = "1.0.0"
+description = "GUI tool to rank, score and prune Javis voice reference clips."
+requires-python = ">=3.10"
+dependencies = [
+    "numpy>=1.24,<2.0",
+    "scipy>=1.11",
+    "soundfile>=0.12",
+    "pyyaml>=6.0",
+    "PySide6>=6.6",
+]
+
+[project.scripts]
+voice-ref-curator = "voice_ref_curator.__main__:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
diff --git a/voice-ref-curator/run.bat b/voice-ref-curator/run.bat
new file mode 100644
index 0000000..4f952e2
--- /dev/null
+++ b/voice-ref-curator/run.bat
@@ -0,0 +1,6 @@
+@echo off
+:: Voice Reference Curator — run with the parent Javis venv
+cd /d "%~dp0"
+call ..\.venv\Scripts\activate.bat
+set PYTHONPATH=%~dp0src
+python -m voice_ref_curator %*
diff --git a/voice-ref-curator/src/voice_ref_curator/__init__.py b/voice-ref-curator/src/voice_ref_curator/__init__.py
new file mode 100644
index 0000000..2c65ecc
--- /dev/null
+++ b/voice-ref-curator/src/voice_ref_curator/__init__.py
@@ -0,0 +1 @@
+"""Voice Reference Curator package."""
diff --git a/voice-ref-curator/src/voice_ref_curator/__main__.py b/voice-ref-curator/src/voice_ref_curator/__main__.py
new file mode 100644
index 0000000..4f06527
--- /dev/null
+++ b/voice-ref-curator/src/voice_ref_curator/__main__.py
@@ -0,0 +1,10 @@
+"""Entry point — launches the GUI."""
+from .gui import run_gui
+
+
+def main() -> None:
+    run_gui()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/voice-ref-curator/src/voice_ref_curator/analyser.py b/voice-ref-curator/src/voice_ref_curator/analyser.py
new file mode 100644
index 0000000..50d638e
--- /dev/null
+++ b/voice-ref-curator/src/voice_ref_curator/analyser.py
@@ -0,0 +1,349 @@
+"""
+Core audio analysis logic for Voice Reference Curator.
+
+Metrics computed per clip
+--------------------------
+duration_s      — clip length in seconds
+rms_db          — mean frame-level RMS in dBFS
+snr_est_db      — SNR estimate: 75th − 10th percentile frame level (dB)
+voiced_ratio    — fraction of frames above the silence threshold
+pitch_mean_hz   — median F0 in voiced frames (normalized autocorrelation)
+pitch_stability — 1 − (std/mean) of voiced F0 values; bounded [0, 1]
+mfcc_vec        — L2-normalised mean MFCC vector (shape: N_MFCC,)
+mfcc_sim        — cosine similarity to the per-profile MFCC centroid
+score           — weighted composite score (higher = better reference clip)
+"""
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+import numpy as np
+import soundfile as sf
+import yaml
+from scipy.fft import dct as scipy_dct
+from scipy.signal import lfilter
+
+if TYPE_CHECKING:
+    from .speaker import NullEmbedder, SpeakerEmbedder
+    _Embedder = Union[SpeakerEmbedder, NullEmbedder]
+
+# ── tuneable constants ────────────────────────────────────────────────────────
+
+FRAME_MS: int = 25             # analysis frame in ms
+HOP_MS: int = 10               # hop in ms
+PITCH_FMIN: float = 70.0       # Hz — minimum voiced pitch
+PITCH_FMAX: float = 400.0      # Hz — maximum voiced pitch
+SILENCE_DB: float = -40.0      # dBFS — frames below this are silent
+PITCH_AC_THRESH: float = 0.30  # min autocorrelation peak to declare voiced
+
+# XTTS ideal reference length is 5–12 s; outside gets a score penalty
+REF_IDEAL_MIN_S: float = 5.0
+REF_IDEAL_MAX_S: float = 12.0
+REF_LEN_PENALTY: float = 0.85
+
+N_MELS: int = 40
+N_MFCC: int = 20
+MEL_FMAX: float = 8000.0
+
+# Composite score weights (must sum to 1.0)
+W_SNR: float = 0.30
+W_VOICED: float = 0.25
+W_PITCH_STAB: float = 0.20
+W_MFCC_SIM: float = 0.25
+
+SNR_NORM_DB: float = 40.0   # 40 dB SNR maps to score 1.0
+
+
+# ── data class ────────────────────────────────────────────────────────────────
+
+@dataclass
+class ClipMetrics:
+    path: Path
+    duration_s: float
+    rms_db: float
+    snr_est_db: float
+    voiced_ratio: float
+    pitch_mean_hz: float
+    pitch_stability: float
+    mfcc_vec: np.ndarray = field(repr=False)
+    mfcc_sim: float = 0.0
+    score: float = 0.0
+    # Speaker-separation fields (populated by compute_speaker_separation)
+    embed_vec: np.ndarray = field(
+        default_factory=lambda: np.zeros(192, dtype=np.float32), repr=False
+    )
+    speaker_label: str = ""   # "primary" | "speaker 2" | … | "" = not computed
+
+
+# ── audio helpers ─────────────────────────────────────────────────────────────
+
+def _load_mono(path: Path) -> tuple[np.ndarray, int]:
+    audio, sr = sf.read(str(path), dtype="float32", always_2d=True)
+    mono = audio.mean(axis=1) if audio.shape[1] > 1 else audio[:, 0]
+    return mono, int(sr)
+
+
+def _rms_db_frames(audio: np.ndarray, sr: int) -> np.ndarray:
+    fl = max(1, int(sr * FRAME_MS / 1000))
+    hl = max(1, int(sr * HOP_MS / 1000))
+    n = max(1, (len(audio) - fl) // hl + 1)
+    out = np.empty(n, dtype=np.float32)
+    for i in range(n):
+        s = i * hl
+        rms = float(np.sqrt(np.mean(audio[s:s + fl] ** 2) + 1e-12))
+        out[i] = 20.0 * math.log10(rms + 1e-12)
+    return out
+
+
+def _snr_db(rms_db: np.ndarray) -> float:
+    if len(rms_db) < 4:
+        return 0.0
+    return float(max(0.0, np.percentile(rms_db, 75) - np.percentile(rms_db, 10)))
+
+
+def _voiced_ratio(rms_db: np.ndarray) -> float:
+    return float(np.mean(rms_db > SILENCE_DB))
+
+
+def _pitch_f0(audio: np.ndarray, sr: int, rms_db: np.ndarray) -> list[float]:
+    """Normalized autocorrelation F0 extraction. Returns voiced Hz values only."""
+    fl = max(1, int(sr * FRAME_MS / 1000))
+    hl = max(1, int(sr * HOP_MS / 1000))
+    min_lag = max(1, int(sr / PITCH_FMAX))
+    max_lag = int(sr / PITCH_FMIN)
+    hann = np.hanning(fl)
+    n = min(len(rms_db), max(1, (len(audio) - fl) // hl + 1))
+    voiced: list[float] = []
+    for i in range(n):
+        if rms_db[i] < SILENCE_DB:
+            continue
+        s = i * hl
+        frame = audio[s:s + fl]
+        if len(frame) < fl:
+            break
+        fft = np.fft.rfft(frame * hann, n=2 * fl)
+        ac = np.fft.irfft(fft * np.conj(fft))[:fl].real
+        if ac[0] < 1e-10:
+            continue
+        ac /= ac[0]
+        region = ac[min_lag:max_lag + 1]
+        if region.size == 0:
+            continue
+        pk = int(np.argmax(region))
+        if region[pk] > PITCH_AC_THRESH:
+            voiced.append(float(sr / (pk + min_lag)))
+    return voiced
+
+
+@lru_cache(maxsize=16)
+def _mel_filterbank(sr: int, n_fft: int) -> np.ndarray:
+    """Build (N_MELS, n_fft//2+1) triangular mel filterbank (cached by sr+n_fft)."""
+    mel_lo = 1127.0 * math.log(1.0 + PITCH_FMIN / 700.0)
+    mel_hi = 1127.0 * math.log(1.0 + min(sr / 2.0, MEL_FMAX) / 700.0)
+    mel_pts = np.linspace(mel_lo, mel_hi, N_MELS + 2)
+    hz_pts = 700.0 * (np.exp(mel_pts / 1127.0) - 1.0)
+    bins = np.floor((n_fft + 1) * hz_pts / sr).astype(int)
+    fb = np.zeros((N_MELS, n_fft // 2 + 1), dtype=np.float32)
+    for m in range(1, N_MELS + 1):
+        lo, ctr, hi = bins[m - 1], bins[m], bins[m + 1]
+        if ctr > lo:
+            for k in range(lo, ctr):
+                fb[m - 1, k] = (k - lo) / (ctr - lo)
+        if hi > ctr:
+            for k in range(ctr, hi):
+                fb[m - 1, k] = (hi - k) / (hi - ctr)
+    return fb
+
+
+def _compute_mfcc(audio: np.ndarray, sr: int) -> np.ndarray:
+    """Mean MFCC vector (N_MFCC,), L2-normalised. Uses scipy DCT; no librosa needed."""
+    fl = max(1, int(sr * FRAME_MS / 1000))
+    hl = max(1, int(sr * HOP_MS / 1000))
+    pre = lfilter([1.0, -0.97], [1.0], audio)
+    fb = _mel_filterbank(sr, fl)
+    hann = np.hanning(fl)
+    n = max(1, (len(pre) - fl) // hl + 1)
+    log_mel_list: list[np.ndarray] = []
+    for i in range(n):
+        s = i * hl
+        frame = pre[s:s + fl]
+        if len(frame) < fl:
+            break
+        spec = np.abs(np.fft.rfft(frame * hann)) ** 2
+        log_mel_list.append(np.log(fb @ spec + 1e-10))
+    if not log_mel_list:
+        return np.zeros(N_MFCC, dtype=np.float32)
+    log_mel = np.stack(log_mel_list, axis=0)                            # (T, N_MELS)
+    mfcc = scipy_dct(log_mel, type=2, n=N_MFCC, axis=1, norm="ortho")  # (T, N_MFCC)
+    mean_vec = mfcc.mean(axis=0).astype(np.float32)
+    norm = float(np.linalg.norm(mean_vec))
+    return mean_vec / norm if norm > 1e-10 else mean_vec
+
+
+# ── per-clip analysis ─────────────────────────────────────────────────────────
+
+def analyse_clip(
+    path: Path,
+    embedder: Optional["_Embedder"] = None,
+) -> Optional[ClipMetrics]:
+    """Compute all metrics for one WAV.
+
+    When *embedder* is provided and available, an ECAPA-TDNN speaker
+    embedding is also computed and stored in ``ClipMetrics.embed_vec``
+    so that :func:`compute_speaker_separation` can cluster clips without
+    re-loading audio from disk.
+
+    Returns None on error or if the clip is too short (< 0.5 s).
+    """
+    try:
+        audio, sr = _load_mono(path)
+    except Exception:
+        return None
+
+    if len(audio) < sr * 0.5:
+        return None
+
+    dur = len(audio) / sr
+    rdb = _rms_db_frames(audio, sr)
+    snr = _snr_db(rdb)
+    vr = _voiced_ratio(rdb)
+    pitches = _pitch_f0(audio, sr, rdb)
+
+    if pitches:
+        pa = np.array(pitches, dtype=np.float32)
+        pmean = float(np.median(pa))
+        pstab = float(max(0.0, 1.0 - float(np.std(pa)) / (pmean + 1e-6)))
+    else:
+        pmean, pstab = 0.0, 0.0
+
+    # Optionally compute speaker embedding while audio is already in memory.
+    if embedder is not None and getattr(embedder, "available", False):
+        embed_vec = embedder.embed(audio, sr)
+    else:
+        embed_vec = np.zeros(192, dtype=np.float32)
+
+    return ClipMetrics(
+        path=path,
+        duration_s=dur,
+        rms_db=float(np.mean(rdb)),
+        snr_est_db=snr,
+        voiced_ratio=vr,
+        pitch_mean_hz=pmean,
+        pitch_stability=pstab,
+        mfcc_vec=_compute_mfcc(audio, sr),
+        embed_vec=embed_vec,
+    )
+
+
+# ── scoring ───────────────────────────────────────────────────────────────────
+
+_SPEAKER_PENALTY: float = 0.45  # score multiplier for non-primary speaker clips
+
+
+def _mfcc_centroid_sim(source_clips: list[ClipMetrics], all_clips: list[ClipMetrics]) -> None:
+    """Compute cosine similarity of every clip in *all_clips* against the MFCC
+    centroid built from *source_clips*.  Mutates ``mfcc_sim`` in place."""
+    non_zero = [c for c in source_clips if np.any(c.mfcc_vec != 0)]
+    if non_zero:
+        stacked = np.stack([c.mfcc_vec for c in non_zero], axis=0)
+        centroid = stacked.mean(axis=0)
+        cn = float(np.linalg.norm(centroid))
+        if cn > 1e-10:
+            centroid /= cn
+        for c in all_clips:
+            c.mfcc_sim = (
+                float(max(0.0, np.dot(c.mfcc_vec, centroid)))
+                if np.any(c.mfcc_vec != 0)
+                else 0.0
+            )
+    else:
+        for c in all_clips:
+            c.mfcc_sim = 0.0
+
+
+def compute_speaker_separation(clips: list[ClipMetrics]) -> None:
+    """Cluster clips by speaker embedding and set ``speaker_label`` on each.
+
+    Expects ``ClipMetrics.embed_vec`` to be populated (non-zero) by
+    :func:`analyse_clip` called with a live ``SpeakerEmbedder``.  When all
+    embeddings are zero (embedder unavailable), the function returns without
+    modifying any clip so that behaviour is identical to the pre-separation
+    baseline.
+
+    Label convention:
+    * ``"primary"``   — largest speaker cluster (target voice)
+    * ``"speaker 2"`` — second-largest cluster
+    * ``"speaker 3"`` — etc.
+    * ``""``          — separation was not run
+    """
+    if not clips:
+        return
+
+    # Skip gracefully when no embeddings were computed.
+    if not any(np.any(c.embed_vec != 0) for c in clips):
+        return
+
+    from .speaker import cluster_speakers
+
+    embeddings = [c.embed_vec for c in clips]
+    labels = cluster_speakers(embeddings)
+
+    for clip, label in zip(clips, labels):
+        clip.speaker_label = "primary" if label == 0 else f"speaker {label + 1}"
+
+
+def compute_scores(clips: list[ClipMetrics]) -> None:
+    """Compute per-profile centroid similarity and final weighted score (mutates in place).
+
+    When speaker separation has been run (``speaker_label`` is set), the MFCC
+    centroid is built exclusively from **primary-speaker** clips.  This means
+    non-primary clips get a naturally lower ``mfcc_sim`` because their spectral
+    profile diverges from the target voice — on top of the explicit
+    ``_SPEAKER_PENALTY`` applied to their raw score.
+    """
+    speaker_labels = {c.speaker_label for c in clips}
+    separation_done = bool(speaker_labels - {""})
+
+    if separation_done:
+        primary_clips = [c for c in clips if c.speaker_label == "primary"]
+        source = primary_clips if primary_clips else clips
+    else:
+        source = clips
+
+    _mfcc_centroid_sim(source, clips)
+
+    for c in clips:
+        snr_s = min(1.0, c.snr_est_db / SNR_NORM_DB)
+        base = (
+            W_SNR * snr_s
+            + W_VOICED * c.voiced_ratio
+            + W_PITCH_STAB * c.pitch_stability
+            + W_MFCC_SIM * c.mfcc_sim
+        )
+        if c.duration_s < REF_IDEAL_MIN_S or c.duration_s > REF_IDEAL_MAX_S:
+            base *= REF_LEN_PENALTY
+        if separation_done and c.speaker_label and c.speaker_label != "primary":
+            base *= _SPEAKER_PENALTY
+        c.score = float(base)
+
+
+# ── voices dir auto-detection ─────────────────────────────────────────────────
+
+def detect_voices_dir() -> Path:
+    """Walk up from this file's location looking for config.yaml with paths.base_dir."""
+    here = Path(__file__).resolve().parent
+    for candidate in [here, here.parent, here.parent.parent, here.parent.parent.parent]:
+        cfg = candidate / "config.yaml"
+        if cfg.exists():
+            try:
+                data = yaml.safe_load(cfg.read_text(encoding="utf-8")) or {}
+                base = data.get("paths", {}).get("base_dir")
+                if base:
+                    return Path(base).expanduser() / "voices"
+            except Exception:
+                pass
+    return Path.home() / ".javis" / "voices"
diff --git a/voice-ref-curator/src/voice_ref_curator/gui.py b/voice-ref-curator/src/voice_ref_curator/gui.py
new file mode 100644
index 0000000..026ea2f
--- /dev/null
+++ b/voice-ref-curator/src/voice_ref_curator/gui.py
@@ -0,0 +1,708 @@
+"""Voice Reference Curator — PySide6 GUI."""
+from __future__ import annotations
+
+import math
+import os
+import shutil
+import sys
+import traceback
+from pathlib import Path
+from typing import Optional
+
+from PySide6.QtCore import Qt, QThread, Signal
+from PySide6.QtGui import QColor, QFont
+from PySide6.QtWidgets import (
+    QAbstractItemView,
+    QApplication,
+    QCheckBox,
+    QComboBox,
+    QDoubleSpinBox,
+    QFileDialog,
+    QGroupBox,
+    QHBoxLayout,
+    QHeaderView,
+    QLabel,
+    QLineEdit,
+    QMainWindow,
+    QMessageBox,
+    QProgressBar,
+    QPushButton,
+    QSpinBox,
+    QStatusBar,
+    QTableWidget,
+    QTableWidgetItem,
+    QVBoxLayout,
+    QWidget,
+)
+
+from .analyser import (
+    ClipMetrics,
+    analyse_clip,
+    compute_scores,
+    compute_speaker_separation,
+    detect_voices_dir,
+)
+
+# ── table column layout ───────────────────────────────────────────────────────
+
+COL_CHECK   = 0
+COL_PROFILE = 1
+COL_SPEAKER = 2
+COL_FILE    = 3
+COL_DUR     = 4
+COL_SNR     = 5
+COL_VOICED  = 6
+COL_PITCH   = 7
+COL_PSTAB   = 8
+COL_SIM     = 9
+COL_SCORE   = 10
+N_COLS      = 11
+
+HEADERS = ["", "Profile", "Speaker", "File", "Dur(s)", "SNR(dB)", "Voiced", "Pitch(Hz)", "PStab", "Sim", "Score"]
+
+_SPEAKER_COLORS = {
+    "primary":   None,        # use score-based colour
+    "speaker 2": QColor(255, 213, 170),   # light orange
+    "speaker 3": QColor(200, 210, 255),   # light blue
+    "speaker 4": QColor(220, 255, 200),   # light green (different from score-green)
+}
+
+
+def _row_color(score: float, speaker_label: str) -> QColor:
+    """Background colour for a table row.  Non-primary speakers get a fixed
+    accent colour so they stand out regardless of their quality score."""
+    if speaker_label and speaker_label != "primary":
+        return _SPEAKER_COLORS.get(speaker_label, QColor(230, 200, 255))  # light purple fallback
+    if score >= 0.72:
+        return QColor(198, 239, 206)   # light green
+    if score >= 0.50:
+        return QColor(255, 235, 156)   # light yellow
+    return QColor(255, 199, 206)       # light red / pink
+
+
+class _NumItem(QTableWidgetItem):
+    """QTableWidgetItem that sorts by its Qt.UserRole float value."""
+
+    def __lt__(self, other: QTableWidgetItem) -> bool:
+        try:
+            return float(self.data(Qt.UserRole)) < float(other.data(Qt.UserRole))
+        except (TypeError, ValueError):
+            return super().__lt__(other)
+
+
+def _num_item(value: float, fmt: str = ".1f") -> _NumItem:
+    item = _NumItem(format(value, fmt))
+    item.setData(Qt.UserRole, value)
+    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
+    item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
+    return item
+
+
+# ── background worker ─────────────────────────────────────────────────────────
+
+class _AnalysisWorker(QThread):
+    progress_total = Signal(int)          # total WAV count
+    progress_step  = Signal(int, str)     # (n_done, current_filename)
+    profile_done   = Signal(str, list)    # (profile_name, list[ClipMetrics]) — scored
+    status_msg     = Signal(str)          # free-form status text for the UI
+    finished       = Signal()
+    error          = Signal(str)
+
+    def __init__(
+        self,
+        voices_dir: Path,
+        profile_filter: Optional[str] = None,
+        use_speaker_sep: bool = False,
+    ):
+        super().__init__()
+        self._voices_dir = voices_dir
+        self._profile_filter = profile_filter
+        self._use_speaker_sep = use_speaker_sep
+        self._cancelled = False
+
+    def cancel(self) -> None:
+        self._cancelled = True
+
+    def run(self) -> None:
+        try:
+            # ── Optional: load speaker embedder ──────────────────────────────
+            embedder = None
+            if self._use_speaker_sep:
+                self.status_msg.emit("Loading ECAPA-TDNN speaker model…")
+                from .speaker import get_embedder
+                embedder = get_embedder()
+                if embedder.available:
+                    self.status_msg.emit("Speaker model ready — analysing clips…")
+                else:
+                    self.status_msg.emit(
+                        "speechbrain not installed — speaker separation disabled"
+                    )
+                    embedder = None
+
+            if self._profile_filter:
+                profile_dirs = [self._voices_dir / self._profile_filter]
+            else:
+                profile_dirs = [
+                    p for p in sorted(self._voices_dir.iterdir()) if p.is_dir()
+                ]
+
+            # Collect all (profile_dir, wav_path) pairs
+            all_items: list[tuple[Path, Path]] = []
+            for pd in profile_dirs:
+                refs = pd / "refs"
+                if refs.is_dir():
+                    for wav in sorted(refs.glob("*.wav")):
+                        all_items.append((pd, wav))
+
+            self.progress_total.emit(len(all_items))
+
+            current_pd: Optional[Path] = None
+            current_clips: list[ClipMetrics] = []
+            done = 0
+
+            for pd, wav in all_items:
+                if self._cancelled:
+                    return
+
+                # New profile boundary — emit the previous batch
+                if pd is not current_pd:
+                    if current_pd is not None and current_clips:
+                        if embedder is not None:
+                            compute_speaker_separation(current_clips)
+                        compute_scores(current_clips)
+                        self.profile_done.emit(current_pd.name, list(current_clips))
+                    current_pd = pd
+                    current_clips = []
+
+                self.progress_step.emit(done, wav.name)
+                m = analyse_clip(wav, embedder)
+                if m is not None:
+                    current_clips.append(m)
+                done += 1
+
+            # Emit the final profile
+            if current_pd is not None and current_clips:
+                if embedder is not None:
+                    compute_speaker_separation(current_clips)
+                compute_scores(current_clips)
+                self.profile_done.emit(current_pd.name, list(current_clips))
+
+            self.finished.emit()
+
+        except Exception:
+            self.error.emit(traceback.format_exc())
+
+
+# ── main window ───────────────────────────────────────────────────────────────
+
+class MainWindow(QMainWindow):
+    def __init__(self) -> None:
+        super().__init__()
+        self.setWindowTitle("Voice Reference Curator")
+        self.resize(1060, 680)
+        self._clips: list[tuple[str, ClipMetrics]] = []  # (profile_name, metrics)
+        self._worker: Optional[_AnalysisWorker] = None
+        self._adding_rows = False
+        self._build_ui()
+        self._auto_detect()
+
+    # ── UI construction ───────────────────────────────────────────────────────
+
+    def _build_ui(self) -> None:
+        root = QWidget()
+        self.setCentralWidget(root)
+        vbox = QVBoxLayout(root)
+        vbox.setSpacing(6)
+        vbox.setContentsMargins(10, 10, 10, 6)
+
+        # ── Config group ──────────────────────────────────────────────────────
+        cfg = QGroupBox("Configuration")
+        cfg_v = QVBoxLayout(cfg)
+        cfg_v.setSpacing(4)
+
+        # Voices directory row
+        dir_row = QHBoxLayout()
+        dir_row.addWidget(QLabel("Voices dir:"))
+        self._dir_edit = QLineEdit()
+        self._dir_edit.setPlaceholderText("Path to Javis voices directory…")
+        dir_row.addWidget(self._dir_edit, 1)
+        btn_browse = QPushButton("Browse…")
+        btn_browse.clicked.connect(self._browse)
+        dir_row.addWidget(btn_browse)
+        btn_auto = QPushButton("Auto-detect")
+        btn_auto.clicked.connect(self._auto_detect)
+        dir_row.addWidget(btn_auto)
+        cfg_v.addLayout(dir_row)
+
+        # Profile + options row
+        opt_row = QHBoxLayout()
+        opt_row.addWidget(QLabel("Profile:"))
+        self._profile_combo = QComboBox()
+        self._profile_combo.setMinimumWidth(160)
+        opt_row.addWidget(self._profile_combo)
+        opt_row.addSpacing(16)
+        opt_row.addWidget(QLabel("Prune worst:"))
+        self._prune_spin = QDoubleSpinBox()
+        self._prune_spin.setRange(0.0, 50.0)
+        self._prune_spin.setSingleStep(0.5)
+        self._prune_spin.setValue(1.0)
+        self._prune_spin.setSuffix(" %")
+        self._prune_spin.setDecimals(1)
+        opt_row.addWidget(self._prune_spin)
+        opt_row.addSpacing(16)
+        self._dry_run_chk = QCheckBox("Dry run (no files moved)")
+        opt_row.addWidget(self._dry_run_chk)
+        opt_row.addSpacing(16)
+        self._speaker_sep_chk = QCheckBox("Separate voices (ECAPA-TDNN)")
+        self._speaker_sep_chk.setToolTip(
+            "Use speechbrain ECAPA-TDNN speaker embeddings to detect and\n"
+            "label multiple voices within each profile's refs/ folder.\n"
+            "Requires: pip install speechbrain torch"
+        )
+        opt_row.addWidget(self._speaker_sep_chk)
+        opt_row.addStretch()
+        self._analyse_btn = QPushButton("▶  Analyse")
+        font = self._analyse_btn.font()
+        font.setBold(True)
+        self._analyse_btn.setFont(font)
+        self._analyse_btn.setMinimumWidth(110)
+        self._analyse_btn.clicked.connect(self._start_analysis)
+        opt_row.addWidget(self._analyse_btn)
+        cfg_v.addLayout(opt_row)
+
+        vbox.addWidget(cfg)
+
+        # ── Progress row ──────────────────────────────────────────────────────
+        prog_row = QHBoxLayout()
+        self._progress = QProgressBar()
+        self._progress.setTextVisible(False)
+        self._progress.setFixedHeight(14)
+        prog_row.addWidget(self._progress, 1)
+        self._prog_label = QLabel("Ready")
+        self._prog_label.setMinimumWidth(220)
+        prog_row.addWidget(self._prog_label)
+        vbox.addLayout(prog_row)
+
+        # ── Results table ─────────────────────────────────────────────────────
+        self._table = QTableWidget(0, N_COLS)
+        self._table.setHorizontalHeaderLabels(HEADERS)
+        self._table.horizontalHeader().setSortIndicatorShown(True)
+        self._table.setSortingEnabled(False)   # enabled after analysis completes
+        self._table.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self._table.setAlternatingRowColors(False)
+        self._table.verticalHeader().setVisible(False)
+        self._table.setEditTriggers(QAbstractItemView.NoEditTriggers)
+        self._table.setShowGrid(True)
+        self._table.cellDoubleClicked.connect(self._open_in_explorer)
+        self._table.itemChanged.connect(self._on_item_changed)
+
+        hh = self._table.horizontalHeader()
+        hh.setSectionResizeMode(COL_CHECK,   QHeaderView.Fixed)
+        hh.setSectionResizeMode(COL_PROFILE, QHeaderView.ResizeToContents)
+        hh.setSectionResizeMode(COL_SPEAKER, QHeaderView.ResizeToContents)
+        hh.setSectionResizeMode(COL_FILE,    QHeaderView.Stretch)
+        for col in (COL_DUR, COL_SNR, COL_VOICED, COL_PITCH, COL_PSTAB, COL_SIM, COL_SCORE):
+            hh.setSectionResizeMode(col, QHeaderView.ResizeToContents)
+        self._table.setColumnWidth(COL_CHECK, 28)
+
+        vbox.addWidget(self._table, 1)
+
+        # ── Action bar ────────────────────────────────────────────────────────
+        act_row = QHBoxLayout()
+
+        btn_worst = QPushButton("Select Worst %")
+        btn_worst.setToolTip("Check the worst N% based on the Prune % setting above")
+        btn_worst.clicked.connect(self._select_worst)
+        act_row.addWidget(btn_worst)
+
+        btn_all = QPushButton("Select All")
+        btn_all.clicked.connect(self._select_all)
+        act_row.addWidget(btn_all)
+
+        btn_none = QPushButton("Deselect All")
+        btn_none.clicked.connect(self._deselect_all)
+        act_row.addWidget(btn_none)
+
+        self._sel_nonprimary_btn = QPushButton("Select Non-primary")
+        self._sel_nonprimary_btn.setToolTip(
+            "Select all clips that were identified as a secondary speaker "
+            "(requires speaker separation to have been run)"
+        )
+        self._sel_nonprimary_btn.clicked.connect(self._select_non_primary)
+        self._sel_nonprimary_btn.setEnabled(False)
+        act_row.addWidget(self._sel_nonprimary_btn)
+
+        act_row.addStretch()
+
+        act_row.addWidget(QLabel("Top:"))
+        self._top_spin = QSpinBox()
+        self._top_spin.setRange(1, 999)
+        self._top_spin.setValue(12)
+        act_row.addWidget(self._top_spin)
+
+        btn_copy = QPushButton("Copy Top Paths")
+        btn_copy.setToolTip("Copy the top-N clip paths to clipboard (paste into config.yaml or scripts)")
+        btn_copy.clicked.connect(self._copy_top_paths)
+        act_row.addWidget(btn_copy)
+
+        act_row.addSpacing(16)
+
+        self._prune_btn = QPushButton("🗑  Prune Selected")
+        self._prune_btn.setEnabled(False)
+        self._prune_btn.clicked.connect(self._prune_selected)
+        act_row.addWidget(self._prune_btn)
+
+        vbox.addLayout(act_row)
+
+        # ── Status bar ────────────────────────────────────────────────────────
+        self._status = QStatusBar()
+        self.setStatusBar(self._status)
+        self._status.showMessage("Load a voices directory and click Analyse.")
+
+    # ── helpers ───────────────────────────────────────────────────────────────
+
+    def _set_status(self, msg: str) -> None:
+        self._status.showMessage(msg)
+
+    def _refresh_profile_combo(self) -> None:
+        vd = Path(self._dir_edit.text().strip())
+        self._profile_combo.clear()
+        self._profile_combo.addItem("All profiles", None)
+        if vd.is_dir():
+            for p in sorted(vd.iterdir()):
+                if p.is_dir():
+                    self._profile_combo.addItem(p.name, p.name)
+
+    def _update_prune_btn(self) -> None:
+        n = self._count_checked()
+        self._prune_btn.setEnabled(n > 0)
+        label = f"🗑  Prune Selected ({n})" if n > 0 else "🗑  Prune Selected"
+        self._prune_btn.setText(label)
+
+    def _count_checked(self) -> int:
+        count = 0
+        for row in range(self._table.rowCount()):
+            item = self._table.item(row, COL_CHECK)
+            if item and item.checkState() == Qt.Checked:
+                count += 1
+        return count
+
+    def _get_checked_rows(self) -> list[int]:
+        return [
+            row for row in range(self._table.rowCount())
+            if (item := self._table.item(row, COL_CHECK)) and item.checkState() == Qt.Checked
+        ]
+
+    # ── actions ───────────────────────────────────────────────────────────────
+
+    def _browse(self) -> None:
+        d = QFileDialog.getExistingDirectory(
+            self, "Select Voices Directory", self._dir_edit.text() or str(Path.home())
+        )
+        if d:
+            self._dir_edit.setText(d)
+            self._refresh_profile_combo()
+
+    def _auto_detect(self) -> None:
+        path = detect_voices_dir()
+        self._dir_edit.setText(str(path))
+        self._refresh_profile_combo()
+        if not path.exists():
+            self._set_status(f"Auto-detected path does not exist yet: {path}")
+
+    def _start_analysis(self) -> None:
+        voices_dir = Path(self._dir_edit.text().strip())
+        if not voices_dir.is_dir():
+            QMessageBox.warning(self, "Directory Not Found",
+                                f"Voices directory not found:\n{voices_dir}")
+            return
+
+        # Stop any running worker
+        if self._worker and self._worker.isRunning():
+            self._worker.cancel()
+            self._worker.wait(3000)
+
+        # Clear previous state
+        self._clips.clear()
+        self._table.blockSignals(True)
+        self._table.setSortingEnabled(False)
+        self._table.setRowCount(0)
+        self._table.blockSignals(False)
+
+        profile_filter: Optional[str] = self._profile_combo.currentData()
+
+        self._analyse_btn.setEnabled(False)
+        self._progress.setValue(0)
+        self._progress.setMaximum(1)
+        self._prog_label.setText("Counting clips…")
+        self._sel_nonprimary_btn.setEnabled(False)
+
+        self._worker = _AnalysisWorker(
+            voices_dir,
+            profile_filter,
+            use_speaker_sep=self._speaker_sep_chk.isChecked(),
+        )
+        self._worker.progress_total.connect(self._on_total)
+        self._worker.progress_step.connect(self._on_step)
+        self._worker.profile_done.connect(self._on_profile_done)
+        self._worker.status_msg.connect(self._set_status)
+        self._worker.finished.connect(self._on_finished)
+        self._worker.error.connect(self._on_error)
+        self._worker.start()
+
+    def _select_all(self) -> None:
+        self._table.blockSignals(True)
+        for row in range(self._table.rowCount()):
+            item = self._table.item(row, COL_CHECK)
+            if item:
+                item.setCheckState(Qt.Checked)
+        self._table.blockSignals(False)
+        self._update_prune_btn()
+
+    def _deselect_all(self) -> None:
+        self._table.blockSignals(True)
+        for row in range(self._table.rowCount()):
+            item = self._table.item(row, COL_CHECK)
+            if item:
+                item.setCheckState(Qt.Unchecked)
+        self._table.blockSignals(False)
+        self._update_prune_btn()
+
+    def _select_non_primary(self) -> None:
+        """Check all clips whose speaker_label is not 'primary'."""
+        self._table.blockSignals(True)
+        for row in range(self._table.rowCount()):
+            spk_item = self._table.item(row, COL_SPEAKER)
+            chk_item = self._table.item(row, COL_CHECK)
+            if spk_item and chk_item:
+                label = spk_item.text()
+                is_non_primary = bool(label) and label != "primary" and label != "—"
+                chk_item.setCheckState(Qt.Checked if is_non_primary else Qt.Unchecked)
+        self._table.blockSignals(False)
+        self._update_prune_btn()
+
+    def _select_worst(self) -> None:
+        n_total = self._table.rowCount()
+        if n_total == 0:
+            return
+        n_prune = max(0, math.floor(n_total * self._prune_spin.value() / 100.0))
+
+        # Collect (score, row) — use UserRole float for correctness after sort
+        scored: list[tuple[float, int]] = []
+        for row in range(n_total):
+            item = self._table.item(row, COL_SCORE)
+            if item:
+                scored.append((float(item.data(Qt.UserRole) or 0.0), row))
+        scored.sort(key=lambda x: x[0])   # ascending — worst first
+        worst = {row for _, row in scored[:n_prune]}
+
+        self._table.blockSignals(True)
+        for row in range(n_total):
+            item = self._table.item(row, COL_CHECK)
+            if item:
+                item.setCheckState(Qt.Checked if row in worst else Qt.Unchecked)
+        self._table.blockSignals(False)
+        self._update_prune_btn()
+
+    def _prune_selected(self) -> None:
+        checked_rows = self._get_checked_rows()
+        if not checked_rows:
+            return
+
+        paths: list[Path] = []
+        for row in checked_rows:
+            item = self._table.item(row, COL_FILE)
+            if item:
+                paths.append(Path(str(item.data(Qt.UserRole))))
+
+        dry_run = self._dry_run_chk.isChecked()
+
+        if not dry_run:
+            reply = QMessageBox.question(
+                self, "Confirm Prune",
+                f"Move {len(paths)} clip(s) to refs/pruned/?\n\n"
+                "They will NOT be deleted — you can restore them manually.",
+                QMessageBox.Yes | QMessageBox.No,
+                QMessageBox.No,
+            )
+            if reply != QMessageBox.Yes:
+                return
+
+        moved, errors = 0, []
+        for p in paths:
+            try:
+                pruned_dir = p.parent / "pruned"
+                if not dry_run:
+                    pruned_dir.mkdir(exist_ok=True)
+                    shutil.move(str(p), str(pruned_dir / p.name))
+                moved += 1
+            except Exception as exc:
+                errors.append(f"{p.name}: {exc}")
+
+        if errors:
+            QMessageBox.warning(self, "Prune Errors", "\n".join(errors))
+
+        if dry_run:
+            self._set_status(f"[DRY RUN] Would have moved {moved} clip(s) to refs/pruned/.")
+        else:
+            self._set_status(f"Pruned {moved} clip(s) → refs/pruned/.")
+            # Remove pruned rows from the table
+            for row in sorted(checked_rows, reverse=True):
+                self._table.removeRow(row)
+            self._update_prune_btn()
+
+    def _copy_top_paths(self) -> None:
+        n = self._top_spin.value()
+        rows_scores: list[tuple[float, int]] = []
+        for row in range(self._table.rowCount()):
+            item = self._table.item(row, COL_SCORE)
+            if item:
+                rows_scores.append((float(item.data(Qt.UserRole) or 0.0), row))
+        rows_scores.sort(key=lambda x: x[0], reverse=True)
+
+        lines: list[str] = []
+        for i, (_, row) in enumerate(rows_scores[:n], 1):
+            item = self._table.item(row, COL_FILE)
+            if item:
+                lines.append(f"{i:2}. {item.data(Qt.UserRole)}")
+
+        if lines:
+            QApplication.clipboard().setText("\n".join(lines))
+            self._set_status(f"Top {len(lines)} paths copied to clipboard.")
+
+    def _open_in_explorer(self, row: int, _col: int) -> None:
+        """Double-click a row → open its parent folder in Explorer/Finder."""
+        item = self._table.item(row, COL_FILE)
+        if not item:
+            return
+        path = Path(str(item.data(Qt.UserRole)))
+        if sys.platform == "win32":
+            os.startfile(str(path.parent))  # noqa: S606
+
+    # ── worker slots ─────────────────────────────────────────────────────────
+
+    def _on_total(self, total: int) -> None:
+        self._progress.setMaximum(max(1, total))
+        self._prog_label.setText(f"0 / {total}")
+
+    def _on_step(self, done: int, name: str) -> None:
+        self._progress.setValue(done)
+        total = self._progress.maximum()
+        self._prog_label.setText(f"{done} / {total}  —  {name}")
+
+    def _on_profile_done(self, profile_name: str, clips: list) -> None:
+        self._adding_rows = True
+        self._table.blockSignals(True)
+        for m in clips:
+            self._clips.append((profile_name, m))
+            self._insert_row(profile_name, m)
+        self._table.blockSignals(False)
+        self._adding_rows = False
+
+    def _on_finished(self) -> None:
+        self._analyse_btn.setEnabled(True)
+        self._table.setSortingEnabled(True)
+        self._table.sortByColumn(COL_SCORE, Qt.DescendingOrder)
+        self._progress.setValue(self._progress.maximum())
+        n_clips = len(self._clips)
+        n_profiles = len({p for p, _ in self._clips})
+        self._prog_label.setText("Analysis complete")
+        # Enable "Select non-primary" button only when separation was run
+        has_labels = any(m.speaker_label for _, m in self._clips)
+        self._sel_nonprimary_btn.setEnabled(has_labels)
+        n_nonprimary = sum(
+            1 for _, m in self._clips
+            if m.speaker_label and m.speaker_label != "primary"
+        )
+        sep_note = f", {n_nonprimary} non-primary clip(s) detected" if has_labels else ""
+        self._set_status(
+            f"{n_clips} clips analysed across {n_profiles} profile(s){sep_note}."
+        )
+        # Auto-select worst N% as a starting suggestion
+        self._select_worst()
+
+    def _on_error(self, msg: str) -> None:
+        self._analyse_btn.setEnabled(True)
+        self._prog_label.setText("Error")
+        QMessageBox.critical(self, "Analysis Error", msg)
+
+    def _on_item_changed(self, item: QTableWidgetItem) -> None:
+        if not self._adding_rows and item.column() == COL_CHECK:
+            self._update_prune_btn()
+
+    # ── row insertion ─────────────────────────────────────────────────────────
+
+    def _insert_row(self, profile_name: str, m: ClipMetrics) -> None:
+        row = self._table.rowCount()
+        self._table.insertRow(row)
+
+        # Checkbox cell
+        chk = QTableWidgetItem()
+        chk.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled)
+        chk.setCheckState(Qt.Unchecked)
+        chk.setData(Qt.UserRole, 0.0)
+        self._table.setItem(row, COL_CHECK, chk)
+
+        # Profile
+        p_item = QTableWidgetItem(profile_name)
+        p_item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
+        self._table.setItem(row, COL_PROFILE, p_item)
+
+        # Speaker label ("—" when separation was not run)
+        spk_text = m.speaker_label if m.speaker_label else "—"
+        spk_item = QTableWidgetItem(spk_text)
+        spk_item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
+        spk_item.setTextAlignment(Qt.AlignCenter | Qt.AlignVCenter)
+        if m.speaker_label and m.speaker_label != "primary":
+            font = spk_item.font()
+            font.setBold(True)
+            spk_item.setFont(font)
+        self._table.setItem(row, COL_SPEAKER, spk_item)
+
+        # File (full path stored in UserRole for later operations)
+        f_item = QTableWidgetItem(m.path.name)
+        f_item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
+        f_item.setData(Qt.UserRole, str(m.path))
+        f_item.setToolTip(str(m.path))
+        self._table.setItem(row, COL_FILE, f_item)
+
+        # Numeric columns
+        self._table.setItem(row, COL_DUR,    _num_item(m.duration_s))
+        self._table.setItem(row, COL_SNR,    _num_item(m.snr_est_db))
+        self._table.setItem(row, COL_VOICED, _num_item(m.voiced_ratio, ".2f"))
+
+        pitch_item = _NumItem(f"{m.pitch_mean_hz:.0f}" if m.pitch_mean_hz > 0 else "N/A")
+        pitch_item.setData(Qt.UserRole, m.pitch_mean_hz)
+        pitch_item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
+        pitch_item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
+        self._table.setItem(row, COL_PITCH, pitch_item)
+
+        self._table.setItem(row, COL_PSTAB,  _num_item(m.pitch_stability, ".2f"))
+        self._table.setItem(row, COL_SIM,    _num_item(m.mfcc_sim, ".2f"))
+        self._table.setItem(row, COL_SCORE,  _num_item(m.score, ".3f"))
+
+        # Row background: non-primary speaker overrides score colour
+        color = _row_color(m.score, m.speaker_label)
+        for col in range(COL_PROFILE, N_COLS):
+            cell = self._table.item(row, col)
+            if cell:
+                cell.setBackground(color)
+
+    # ── cleanup ───────────────────────────────────────────────────────────────
+
+    def closeEvent(self, event) -> None:
+        if self._worker and self._worker.isRunning():
+            self._worker.cancel()
+            self._worker.wait(3000)
+        event.accept()
+
+
+# ── application entry point ───────────────────────────────────────────────────
+
+def run_gui() -> None:
+    app = QApplication.instance() or QApplication(sys.argv)
+    app.setApplicationName("Voice Reference Curator")
+    app.setApplicationVersion("1.0")
+    w = MainWindow()
+    w.show()
+    sys.exit(app.exec())
diff --git a/voice-ref-curator/src/voice_ref_curator/speaker.py b/voice-ref-curator/src/voice_ref_curator/speaker.py
new file mode 100644
index 0000000..f029547
--- /dev/null
+++ b/voice-ref-curator/src/voice_ref_curator/speaker.py
@@ -0,0 +1,228 @@
+"""Standalone speaker embedder for Voice Reference Curator.
+
+Uses speechbrain ECAPA-TDNN to produce speaker-level embeddings and provides
+agglomerative clustering to separate multiple voices within a ``refs/`` folder.
+
+Falls back to ``NullEmbedder`` silently when speechbrain / torch is not
+installed, leaving analysis behaviour identical to the baseline version.
+"""
+from __future__ import annotations
+
+import logging
+from collections import Counter
+from pathlib import Path
+from typing import Optional, Union
+
+import numpy as np
+
+log = logging.getLogger("voice_ref_curator.speaker")
+
+_MODEL_SOURCE = "speechbrain/spkrec-ecapa-voxceleb"
+_TARGET_RMS: float = 0.05
+_MIN_EMBED_S: float = 0.5
+
+# Reuse the Javis model cache when available so the model need not be
+# downloaded twice if Javis is already installed on the same machine.
+_DEFAULT_CACHE_DIR = Path.home() / ".javis" / "models" / "ecapa-tdnn"
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+def _l2(v: np.ndarray) -> np.ndarray:
+    n = np.linalg.norm(v)
+    return v / n if n > 0 else v
+
+
+def _cosine(a: np.ndarray, b: np.ndarray) -> float:
+    na, nb = np.linalg.norm(a), np.linalg.norm(b)
+    if na == 0 or nb == 0:
+        return 0.0
+    return float(np.dot(a, b) / (na * nb))
+
+
+def _normalize_rms(audio: np.ndarray) -> np.ndarray:
+    rms = float(np.sqrt(np.mean(audio.astype(np.float64) ** 2)))
+    if rms < 1e-8:
+        return audio
+    return (audio * (_TARGET_RMS / rms)).astype(np.float32)
+
+
+# ── null fallback ─────────────────────────────────────────────────────────────
+
+class NullEmbedder:
+    """Silent no-op used when speechbrain / torch is unavailable."""
+
+    available: bool = False
+
+    def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:  # noqa: ARG002
+        return np.zeros(192, dtype=np.float32)
+
+
+# ── main embedder ─────────────────────────────────────────────────────────────
+
+class SpeakerEmbedder:
+    """ECAPA-TDNN speaker embedder using speechbrain (no Javis Config needed)."""
+
+    available: bool = False
+
+    def __init__(self, cache_dir: Optional[Path] = None) -> None:
+        self._cache_dir = cache_dir or _DEFAULT_CACHE_DIR
+        self._encoder = None
+
+    # ── lifecycle ─────────────────────────────────────────────────────────────
+
+    def load(self) -> bool:
+        """Download / load the ECAPA-TDNN model. Returns True on success."""
+        try:
+            import os
+            import warnings
+
+            os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+            warnings.filterwarnings(
+                "ignore",
+                message=r".*speechbrain\.pretrained.*deprecated.*",
+                category=UserWarning,
+            )
+            warnings.filterwarnings(
+                "ignore",
+                message=r".*Requested Pretrainer collection using symlinks.*",
+                category=UserWarning,
+            )
+
+            from speechbrain.inference.classifiers import EncoderClassifier
+
+            device = "cuda:0" if self._try_cuda() else "cpu"
+            self._cache_dir.mkdir(parents=True, exist_ok=True)
+            log.info(f"[speaker] loading ECAPA-TDNN on {device} → {self._cache_dir}")
+            self._encoder = EncoderClassifier.from_hparams(
+                source=_MODEL_SOURCE,
+                savedir=str(self._cache_dir),
+                run_opts={"device": device},
+            )
+            self.available = True
+            log.info("[speaker] ECAPA-TDNN ready")
+            return True
+        except Exception as exc:
+            log.warning(
+                f"[speaker] speechbrain unavailable ({exc}) — "
+                "speaker separation disabled"
+            )
+            return False
+
+    @staticmethod
+    def _try_cuda() -> bool:
+        try:
+            import torch
+            return torch.cuda.is_available()
+        except Exception:
+            return False
+
+    # ── embedding ─────────────────────────────────────────────────────────────
+
+    def embed(self, audio_np: np.ndarray, sr: int) -> np.ndarray:
+        """Return an L2-normalised ECAPA-TDNN embedding for *audio_np*."""
+        if self._encoder is None:
+            return np.zeros(192, dtype=np.float32)
+
+        import torch
+
+        dur = len(audio_np) / sr
+        if dur < _MIN_EMBED_S:
+            log.warning(
+                f"[speaker] clip too short ({dur:.2f}s) — embedding may be unreliable"
+            )
+
+        # ECAPA-TDNN was trained at 16 kHz; resample if needed.
+        if sr != 16000:
+            from fractions import Fraction
+            import scipy.signal
+
+            frac = Fraction(16000, sr).limit_denominator(1000)
+            audio_np = scipy.signal.resample_poly(
+                audio_np, frac.numerator, frac.denominator
+            ).astype(np.float32)
+
+        audio_np = _normalize_rms(audio_np)
+        wav = torch.tensor(audio_np.astype(np.float32)).unsqueeze(0)  # [1, T]
+        wav_lens = torch.tensor([1.0])
+        with torch.no_grad():
+            emb = self._encoder.encode_batch(wav, wav_lens)  # [1, 1, D]
+        return _l2(emb.squeeze().cpu().numpy())
+
+
+# ── clustering ────────────────────────────────────────────────────────────────
+
+def cluster_speakers(
+    embeddings: list[np.ndarray],
+    threshold: float = 0.20,
+) -> list[int]:
+    """Group speaker embeddings with single-linkage agglomerative clustering.
+
+    Parameters
+    ----------
+    embeddings:
+        L2-normalised ECAPA-TDNN embedding vectors (one per audio clip).
+    threshold:
+        Cosine *distance* threshold (= 1 − cosine similarity).  Pairs whose
+        distance is below this value are merged into the same cluster.
+        ``0.20`` works well for same-speaker across microphone / session
+        variation; increase to ``0.30`` if the target speaker has high
+        within-session variability.
+
+    Returns
+    -------
+    Integer cluster labels (one per input embedding).  **Label 0 always
+    identifies the largest cluster** (the "primary speaker"); minority
+    clusters are numbered 1, 2, … in descending size order.
+    """
+    n = len(embeddings)
+    if n == 0:
+        return []
+    if n == 1:
+        return [0]
+
+    # Pairwise cosine-distance matrix
+    dist = np.zeros((n, n), dtype=np.float32)
+    for i in range(n):
+        for j in range(i + 1, n):
+            d = 1.0 - max(0.0, _cosine(embeddings[i], embeddings[j]))
+            dist[i, j] = d
+            dist[j, i] = d
+
+    # Single-linkage agglomeration: merge any two clusters whose minimum
+    # pairwise distance falls below *threshold*.
+    labels: list[int] = list(range(n))
+    changed = True
+    while changed:
+        changed = False
+        unique = sorted(set(labels))
+        for ci in unique:
+            if changed:
+                break
+            for cj in unique:
+                if ci >= cj:
+                    continue
+                idxs_i = [k for k, lb in enumerate(labels) if lb == ci]
+                idxs_j = [k for k, lb in enumerate(labels) if lb == cj]
+                min_d = float(np.min(dist[np.ix_(idxs_i, idxs_j)]))
+                if min_d < threshold:
+                    labels = [ci if lb == cj else lb for lb in labels]
+                    changed = True
+                    break
+
+    # Renumber so the largest cluster becomes 0
+    counts = Counter(labels)
+    rank = {old: new for new, (old, _) in enumerate(counts.most_common())}
+    return [rank[lb] for lb in labels]
+
+
+# ── factory ───────────────────────────────────────────────────────────────────
+
+def get_embedder(
+    cache_dir: Optional[Path] = None,
+) -> Union[SpeakerEmbedder, NullEmbedder]:
+    """Load and return a ``SpeakerEmbedder``; fall back to ``NullEmbedder``."""
+    e = SpeakerEmbedder(cache_dir)
+    if e.load():
+        return e
+    return NullEmbedder()