feat: expose truthful inline realtime voice lifecycle

tests/unit/test_voice_settings_ui.py

@@ -40,8 +40,8 @@ def test_english_voice_search_selection_and_retry():
         page.add_init_script("localStorage.setItem('preferred_voice', 'en-US-Ava:DragonHDLatestNeural')")
         page.goto('https://hope.test/'); page.wait_for_load_state('networkidle')
         page.locator('#drawerOpen').click()
-        page.locator('#openVoice').click()
+        page.locator('#accountMenuBtn').click()
-        page.locator('#settingsBtn').click()
+        page.locator('[data-settings-target="preferences"]').click()
         expect(page.locator('#voiceList [aria-checked=true]')).to_contain_text('Ava')
         expect(page.locator('#voiceCatalogStatus')).to_contain_text('3 English voices')
         page.locator('#voiceSearch').fill('en-GB')
@@ -53,9 +53,9 @@ def test_english_voice_search_selection_and_retry():
         page.locator('#voiceSearch').fill('')
         assert page.evaluate('document.documentElement.scrollWidth <= innerWidth')
         page.screenshot(path='/tmp/hope-english-voices-mobile.png', full_page=True)
-        page.locator('#closeSheetBtn').click()
+        page.locator('#settingsClose').click()
         available = False
-        page.locator('#settingsBtn').click()
+        page.evaluate("HopeSettings.open('preferences')")
         expect(page.locator('#retryVoices')).to_be_visible()
         expect(page.locator('#voiceList [role=radio]')).to_have_count(3)
         available = True
@@ -63,3 +63,72 @@ def test_english_voice_search_selection_and_retry():
         expect(page.locator('#retryVoices')).to_be_hidden()
         assert errors == []
         browser.close()
+
+
+# Synthetic transport tests browser wiring only, not actual WebRTC/audio quality.
+VOICE_SDK = r"""
+(() => {
+const events=['ParticipantConnected','ParticipantDisconnected','TrackSubscribed','TrackUnsubscribed','AudioPlaybackStatusChanged','DataReceived','ConnectionQualityChanged','Disconnected','TrackMuted','TrackUnmuted','LocalTrackUnpublished'];
+class Room {
+ constructor(){this.handlers={};this.state='disconnected';this.remoteParticipants=new Map();this.localParticipant={isMicrophoneEnabled:false,getTrackPublication:()=>null,publishData:async()=>{},setMicrophoneEnabled:async enabled=>{if(enabled && window.denyMic)throw Error('Permission denied');this.localParticipant.isMicrophoneEnabled=enabled;}};window.testRoom=this;}
+ on(name,fn){(this.handlers[name]||=[]).push(fn);return this;}
+ emit(name,...args){(this.handlers[name]||[]).forEach(fn=>fn(...args));}
+ registerTextStreamHandler(){} async startAudio(){}
+ async disconnect(){this.state='disconnected';this.localParticipant.isMicrophoneEnabled=false;this.emit('Disconnected');}
+ async connect(){this.state='connected';const agent={identity:'agent',isAgent:true};this.remoteParticipants.set('agent',agent);setTimeout(()=>this.emit('DataReceived',new TextEncoder().encode(JSON.stringify({type:'agent_ready'})),agent,null,'conversation-control'),20);}
+}
+window.LivekitClient={Room,RoomEvent:Object.fromEntries(events.map(x=>[x,x])),Track:{Source:{Microphone:'microphone'},Kind:{Audio:'audio'}}};
+})();
+"""
+
+
+def test_voice_api_lifecycle_and_denied_permission():
+    account = {'id': 'alice', 'username': 'alice', 'display_name': 'Alice', 'role': 'user', 'status': 'ready'}
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        errors = []
+        page.on('pageerror', lambda error: errors.append(str(error)))
+        def route(req):
+            url = req.request.url
+            if url.endswith('/livekit-client.umd.js'):
+                req.fulfill(body=VOICE_SDK, content_type='application/javascript')
+            elif '/api/auth/me' in url:
+                req.fulfill(json={'user': account})
+            elif url.endswith('/token'):
+                req.fulfill(json={'token': 'synthetic-token', 'conversation_id': 'voice-cid'})
+            elif url.endswith('/voices'):
+                req.fulfill(json={'voices': [], 'catalog_available': False})
+            elif '/api/settings/model' in url:
+                req.fulfill(json={'settings': {'model': 'auto', 'thinking_level': 'none'}, 'models': [], 'catalog_available': False})
+            elif '/api/' in url:
+                req.fulfill(json={'conversations': [], 'messages': [], 'files': [], 'data': []})
+            else:
+                asset = ROOT / 'web' / (url.split('/', 3)[-1].split('?')[0] or 'index.html')
+                types = {'.html': 'text/html', '.js': 'application/javascript', '.css': 'text/css', '.svg': 'image/svg+xml'}
+                if asset.is_file():
+                    req.fulfill(body=asset.read_bytes(), content_type=types.get(asset.suffix, 'application/octet-stream'))
+                else:
+                    req.fulfill(status=404)
+        page.route('https://hope.test/**', route)
+        page.goto('https://hope.test/'); page.wait_for_load_state('networkidle')
+        page.evaluate("window.voiceEvents=[];window.boundEvents=[];addEventListener('hope:voice-state',e=>voiceEvents.push(e.detail));addEventListener('hope:voice-conversation-bound',e=>boundEvents.push(e.detail));")
+        assert page.evaluate('HopeVoice.state().ready') is False
+        page.evaluate('HopeVoice.start()')
+        page.wait_for_function('HopeVoice.state().ready')
+        state = page.evaluate('HopeVoice.state()')
+        assert state['connected'] and state['agent_connected'] and state['microphone_enabled']
+        assert state['conversation_id'] == 'voice-cid'
+        assert page.evaluate('boundEvents[0].conversation_id') == 'voice-cid'
+        page.evaluate('HopeVoice.mute()')
+        assert page.evaluate('HopeVoice.state().muted && !HopeVoice.state().microphone_enabled && !HopeVoice.state().ready')
+        page.evaluate('HopeVoice.mute()')
+        assert page.evaluate('HopeVoice.state().ready')
+        page.evaluate("dispatchEvent(new CustomEvent('hope:conversation-selected',{detail:{conversation_id:'other-cid'}}))")
+        assert page.evaluate('!HopeVoice.state().active && !HopeVoice.state().connected')
+        assert page.evaluate('voiceEvents.at(-1).conversation_id') == 'voice-cid'
+        page.evaluate('window.denyMic=true;HopeVoice.start().catch(()=>{})')
+        page.wait_for_function("document.getElementById('status').textContent.includes('Permission denied')")
+        assert page.evaluate('!HopeVoice.state().active && !HopeVoice.state().ready && !HopeVoice.state().microphone_enabled')
+        assert errors == []
+        browser.close()

web/app.js

@@ -29,6 +29,11 @@ let userSpeaking = false;        // local VAD-ish flag from mic level
 let wakeLock = null;
 let conversationMode = localStorage.getItem(MODE_KEY) === "text" ? "text" : "voice";
 let connectionPromise = null;
+let connectionEpoch = 0;
+let tokenAbort = null;
+let voiceConversationId = null;
+let voiceSessionActive = false;
+let lastVoiceState = "";
 
 function escapeHtml(s) {
     const d = document.createElement("div");
@@ -133,6 +138,7 @@ async function setConversationMode(mode) {
     }
 
     if (!room || !agentReady || !room.localParticipant) return;
+    const modeRoom = room;
     sendModeMessage(mode);
     if (mode === "text") {
         await room.localParticipant.setMicrophoneEnabled(false);
@@ -148,7 +154,9 @@ async function setConversationMode(mode) {
         startVisualLoop();
         try {
             await room.startAudio();
-            await room.localParticipant.setMicrophoneEnabled(true);
+            await modeRoom.localParticipant.setMicrophoneEnabled(true);
+            if (modeRoom !== room) { await modeRoom.localParticipant.setMicrophoneEnabled(false); return; }
+            micMuted = !microphoneEnabled();
             setupMicAnalyser();
             statusEl.textContent = "Secure room · Microphone on";
         } catch (err) {
@@ -160,6 +168,7 @@ async function setConversationMode(mode) {
             statusEl.textContent = "Voice mode · Microphone unavailable";
         }
     }
+    updateMuteIcon(); setState(uiState);
 }
 
 voiceModeBtn.addEventListener("click", () => setConversationMode("voice").catch(console.error));
@@ -243,6 +252,8 @@ function buildLedWall() {
 // Renders one frame of the LED wall. `level` is 0..1 (agent audio when speaking,
 // mic when listening). `mode` picks the palette/gradient by conversation state.
 function renderLedWall(level, mode) {
+    const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
+    if (reducedMotion) { level = 0; mode = "idle"; }
     ledWallTime += 1;
     ledWallEl.dataset.mode = mode;
     const n = ledDots.length;
@@ -279,8 +290,8 @@ function renderLedWall(level, mode) {
 
         vis = Math.max(0.02, Math.min(1, vis));
         // Dot grows toward the center and sits on a subtle 4px grid inset.
-        const px = (2 * size).toFixed(1);
+
-        dot.style.setProperty("--sz", `${px}px`);
+        dot.style.transform = `scale(${size.toFixed(2)})`;
         dot.style.opacity = vis.toFixed(3);
     }
 }
@@ -330,9 +341,13 @@ const STATE_LABELS = {
 };
 
 function setState(next) {
+    if (["listening", "speaking"].includes(next) && conversationMode === "voice" && !microphoneEnabled()) next = voiceSessionActive ? "connecting" : "idle";
     uiState = next;
     ledWallEl.dataset.state = next;
-    stateLabelEl.textContent = STATE_LABELS[conversationMode][next] || next;
+    publishVoiceState();
+    stateLabelEl.textContent = conversationMode === "voice" && voiceSessionActive && agentReady && micMuted
+        ? "Microphone muted"
+        : STATE_LABELS[conversationMode][next] || next;
 
     // Thinking timer: show the "thinking..." chip if it lingers past 2s
     if (next === "thinking") {
@@ -358,7 +373,7 @@ updateModeUI();
 
 function inferState() {
     if (!room || !room.localParticipant) return "idle";
-    if (!agentParticipant) return "connecting";
+    if (!agentParticipant || !agentReady || !microphoneEnabled()) return "connecting";
     readMicLevel();
     readAgentAudioLevel();
     userSpeaking = micLevel > 0.035;
@@ -565,6 +580,7 @@ function handleDataPacket(payload, participant, topic) {
         if (!HopeAuth.user || !participant || !isAgent(participant)) return;
         if (msg.type === "agent_ready" && topic === "conversation-control") {
             agentReady = true;
+            publishVoiceState();
             sendVoiceMessage(currentVoiceId());
         } else if (msg.type === "transcript" && topic === "transcript") {
             if (!["agent", "user"].includes(msg.role) || typeof msg.text !== "string") return;
@@ -833,6 +849,7 @@ function openSheet() {
     if (!HopeAuth.user) return;
     renderVoiceList();
     loadVoiceCatalog();
+    if (window.HopeSettings) { window.HopeSettings.open("preferences"); return; }
     settingsSheet.hidden = false; settingsSheet.inert = false;
     document.getElementById("authenticatedApp").inert = true;
     sheetBackdrop.hidden = false;
@@ -844,6 +861,7 @@ function openSheet() {
 }
 
 function closeSheet() {
+    if (window.HopeSettings) { window.HopeSettings.close(); return; }
     settingsSheet.classList.remove("open");
     settingsSheet.hidden = true; settingsSheet.inert = true;
     document.getElementById("authenticatedApp").inert = false;
@@ -914,6 +932,7 @@ function findAgent(rm) {
 function markAgentFound(rm, participant) {
     if (rm !== room || !HopeAuth.user) return;
     agentParticipant = participant;
+    publishVoiceState();
     statusEl.textContent = conversationMode === "text"
         ? "Hope joined · Text chat ready"
         : "Hope joined · Preparing microphone…";
@@ -934,6 +953,14 @@ async function createRoom(token, attempt) {
         if (newRoom === room && isAgent(participant)) markAgentFound(newRoom, participant);
     });
 
+    newRoom.on(RoomEvent.ParticipantDisconnected, participant => {
+        if (newRoom === room && participant === agentParticipant) {
+            stopConversation(); statusEl.textContent = "Hope disconnected. Start again to reconnect.";
+        }
+    });
+    newRoom.on(RoomEvent.TrackMuted, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); publishVoiceState(); } });
+    newRoom.on(RoomEvent.TrackUnmuted, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); publishVoiceState(); } });
+    newRoom.on(RoomEvent.LocalTrackUnpublished, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); updateMuteIcon(); } });
     // Attach and play incoming audio tracks (agent TTS)
     newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
         if (newRoom === room && isAgent(participant) && track.kind === LivekitClient.Track.Kind.Audio) {
@@ -992,16 +1019,33 @@ async function createRoom(token, attempt) {
     });
     await newRoom.connect(LIVEKIT_URL, token, { autoSubscribe: true });
     checkAttempt(attempt);
+    publishVoiceState();
     const existing = findAgent(newRoom);
     if (existing) markAgentFound(newRoom, existing);
     return newRoom;
 }
 
-let connectionEpoch = 0;
+function microphoneEnabled() {
-let tokenAbort = null;
+    const local = room?.localParticipant;
-let voiceConversationId = null;
+    if (!local || !local.isMicrophoneEnabled) return false;
-function publishVoiceState(active) {
+    const track = local.getTrackPublication?.(LivekitClient.Track.Source.Microphone)?.track?.mediaStreamTrack;
-    window.dispatchEvent(new CustomEvent('hope:voice-state', { detail: { active, conversation_id: voiceConversationId } }));
+    return !track || (track.enabled && track.readyState !== "ended");
+}
+function voiceState() {
+    const connected = !!room && room.state === "connected";
+    const microphone_enabled = connected && microphoneEnabled();
+    const agent_connected = connected && !!agentParticipant;
+    const ready = agent_connected && agentReady && microphone_enabled && !micMuted;
+    return { active: voiceSessionActive, state: voiceSessionActive ? (ready ? uiState : (agent_connected && agentReady && micMuted ? "muted" : "connecting")) : "idle",
+        ready, muted: micMuted, connected, agent_connected, microphone_enabled,
+        conversation_id: voiceConversationId, cid: voiceConversationId };
+}
+function publishVoiceState() {
+    const detail = voiceState();
+    const signature = JSON.stringify(detail);
+    if (signature === lastVoiceState) return;
+    lastVoiceState = signature;
+    window.dispatchEvent(new CustomEvent('hope:voice-state', { detail }));
 }
 window.addEventListener('hope:conversation-selected', event => {
     if ((room || connectionPromise) && event.detail?.conversation_id !== voiceConversationId) stopConversation();
@@ -1028,7 +1072,7 @@ function cleanupRoom() {
     document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove());
     document.getElementById("audioUnlockBtn")?.remove();
     muteBtn.disabled = true; micMuted = false; updateMuteIcon();
-    publishVoiceState(false); voiceConversationId = null;
+    voiceSessionActive = false; publishVoiceState();
 }
 function stopConversation() {
     connectionEpoch++; tokenAbort?.abort(); tokenAbort = null; connectionPromise = null;
@@ -1044,7 +1088,7 @@ async function connectConversation() {
     const attempt = ++connectionEpoch;
     if (window.HopeChat?.running) throw new Error('Wait for the current response or stop it before starting voice.');
     voiceConversationId = window.HopeChat?.conversationId || null;
-    publishVoiceState(true);
+    voiceSessionActive = true; publishVoiceState();
     startBtn.disabled = true; stopBtn.disabled = false;
     const promise = (async () => {
         try {
@@ -1079,9 +1123,16 @@ async function connectConversation() {
     return promise;
 }
 
-startBtn.addEventListener("click", () => {
+async function startVoice() {
-    connectConversation().catch(() => {});
+    if (window.HopeChat?.running) throw new Error("Wait for the current response or stop it before starting voice.");
-});
+    await setConversationMode("voice");
+    return connectConversation();
+}
+window.HopeVoiceSettings = Object.freeze({ load: () => { renderVoiceList(); return Promise.all([loadVoiceCatalog(), loadModelSettings()]); } });
+window.HopeVoice = Object.freeze({ start: startVoice, stop: stopConversation, mute: toggleMute, state: voiceState,
+    loadPreferences: () => Promise.all([loadVoiceCatalog(), loadModelSettings()]) });
+window.addEventListener("hope:start-voice", () => startVoice().catch(error => { statusEl.textContent = error.message; }));
+startBtn.addEventListener("click", () => startVoice().catch(error => { statusEl.textContent = error.message; }));
 
 // ── Text chat ──────────────────────────────────────────────────────────────
 function resizeChatInput() {
@@ -1130,20 +1181,25 @@ chatComposer.addEventListener("submit", async (event) => {
 stopBtn.addEventListener("click", stopConversation);
 
 // ── Mic mute toggle ─────────────────────────────────────────────────────────
-muteBtn.addEventListener("click", async () => {
+async function toggleMute() {
-    if (!room || !room.localParticipant) return;
+    if (!room || !agentReady || !room.localParticipant || conversationMode !== "voice") return;
-    const nextMuted = !micMuted;
+    const active = room;
+    const nextMuted = microphoneEnabled();
     try {
-        await room.localParticipant.setMicrophoneEnabled(!nextMuted);
+        await active.localParticipant.setMicrophoneEnabled(!nextMuted);
-        micMuted = nextMuted;
+        if (active !== room) return;
+        micMuted = !microphoneEnabled();
         updateMuteIcon();
     } catch (err) {
         console.warn("Microphone toggle failed:", err);
+        micMuted = !microphoneEnabled(); updateMuteIcon();
         statusEl.textContent = "Microphone unavailable in this browser";
     }
-});
+}
+muteBtn.addEventListener("click", toggleMute);
 
 function updateMuteIcon() {
+    publishVoiceState();
     const muted = muteBtn.classList.toggle("muted", micMuted);
     muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone");
     muteBtn.title = muted ? "Unmute microphone" : "Mute microphone";
@@ -1156,7 +1212,8 @@ async function fetchToken(attempt) {
     const data = await HopeAuth.request("/token", { method: "POST", body: JSON.stringify(voiceConversationId ? { conversation_id: voiceConversationId } : {}), signal: tokenAbort.signal });
     checkAttempt(attempt);
     voiceConversationId = data.conversation_id || voiceConversationId;
-    publishVoiceState(true);
+    if (voiceConversationId) window.dispatchEvent(new CustomEvent('hope:voice-conversation-bound', { detail: { conversation_id: voiceConversationId } }));
+    publishVoiceState();
     if (!data.token || typeof data.token !== "string") throw new Error("Token endpoint did not return a room token");
     return data;
 }
@@ -1183,7 +1240,7 @@ function sendModeMessage(mode, targetRoom) {
 }
 
 HopeAuth.onChange((user, changed) => {
-    if (changed || !user || user.status !== "ready") stopConversation();
+    if (changed || !user || user.status !== "ready") { stopConversation(); voiceConversationId = null; publishVoiceState(); }
     if (changed || settingsAccount !== (user ? String(user.id) : null)) resetModelSettings(user);
     if (changed) {
         voiceCatalogGeneration++;