diff --git a/tests/unit/test_voice_settings_ui.py b/tests/unit/test_voice_settings_ui.py index b5cae48..c43c0f9 100644 --- a/tests/unit/test_voice_settings_ui.py +++ b/tests/unit/test_voice_settings_ui.py @@ -40,8 +40,8 @@ def test_english_voice_search_selection_and_retry(): page.add_init_script("localStorage.setItem('preferred_voice', 'en-US-Ava:DragonHDLatestNeural')") page.goto('https://hope.test/'); page.wait_for_load_state('networkidle') page.locator('#drawerOpen').click() - page.locator('#openVoice').click() - page.locator('#settingsBtn').click() + page.locator('#accountMenuBtn').click() + page.locator('[data-settings-target="preferences"]').click() expect(page.locator('#voiceList [aria-checked=true]')).to_contain_text('Ava') expect(page.locator('#voiceCatalogStatus')).to_contain_text('3 English voices') page.locator('#voiceSearch').fill('en-GB') @@ -53,9 +53,9 @@ def test_english_voice_search_selection_and_retry(): page.locator('#voiceSearch').fill('') assert page.evaluate('document.documentElement.scrollWidth <= innerWidth') page.screenshot(path='/tmp/hope-english-voices-mobile.png', full_page=True) - page.locator('#closeSheetBtn').click() + page.locator('#settingsClose').click() available = False - page.locator('#settingsBtn').click() + page.evaluate("HopeSettings.open('preferences')") expect(page.locator('#retryVoices')).to_be_visible() expect(page.locator('#voiceList [role=radio]')).to_have_count(3) available = True @@ -63,3 +63,72 @@ def test_english_voice_search_selection_and_retry(): expect(page.locator('#retryVoices')).to_be_hidden() assert errors == [] browser.close() + + +# Synthetic transport tests browser wiring only, not actual WebRTC/audio quality. +VOICE_SDK = r""" +(() => { +const events=['ParticipantConnected','ParticipantDisconnected','TrackSubscribed','TrackUnsubscribed','AudioPlaybackStatusChanged','DataReceived','ConnectionQualityChanged','Disconnected','TrackMuted','TrackUnmuted','LocalTrackUnpublished']; +class Room { + constructor(){this.handlers={};this.state='disconnected';this.remoteParticipants=new Map();this.localParticipant={isMicrophoneEnabled:false,getTrackPublication:()=>null,publishData:async()=>{},setMicrophoneEnabled:async enabled=>{if(enabled && window.denyMic)throw Error('Permission denied');this.localParticipant.isMicrophoneEnabled=enabled;}};window.testRoom=this;} + on(name,fn){(this.handlers[name]||=[]).push(fn);return this;} + emit(name,...args){(this.handlers[name]||[]).forEach(fn=>fn(...args));} + registerTextStreamHandler(){} async startAudio(){} + async disconnect(){this.state='disconnected';this.localParticipant.isMicrophoneEnabled=false;this.emit('Disconnected');} + async connect(){this.state='connected';const agent={identity:'agent',isAgent:true};this.remoteParticipants.set('agent',agent);setTimeout(()=>this.emit('DataReceived',new TextEncoder().encode(JSON.stringify({type:'agent_ready'})),agent,null,'conversation-control'),20);} +} +window.LivekitClient={Room,RoomEvent:Object.fromEntries(events.map(x=>[x,x])),Track:{Source:{Microphone:'microphone'},Kind:{Audio:'audio'}}}; +})(); +""" + + +def test_voice_api_lifecycle_and_denied_permission(): + account = {'id': 'alice', 'username': 'alice', 'display_name': 'Alice', 'role': 'user', 'status': 'ready'} + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + errors = [] + page.on('pageerror', lambda error: errors.append(str(error))) + def route(req): + url = req.request.url + if url.endswith('/livekit-client.umd.js'): + req.fulfill(body=VOICE_SDK, content_type='application/javascript') + elif '/api/auth/me' in url: + req.fulfill(json={'user': account}) + elif url.endswith('/token'): + req.fulfill(json={'token': 'synthetic-token', 'conversation_id': 'voice-cid'}) + elif url.endswith('/voices'): + req.fulfill(json={'voices': [], 'catalog_available': False}) + elif '/api/settings/model' in url: + req.fulfill(json={'settings': {'model': 'auto', 'thinking_level': 'none'}, 'models': [], 'catalog_available': False}) + elif '/api/' in url: + req.fulfill(json={'conversations': [], 'messages': [], 'files': [], 'data': []}) + else: + asset = ROOT / 'web' / (url.split('/', 3)[-1].split('?')[0] or 'index.html') + types = {'.html': 'text/html', '.js': 'application/javascript', '.css': 'text/css', '.svg': 'image/svg+xml'} + if asset.is_file(): + req.fulfill(body=asset.read_bytes(), content_type=types.get(asset.suffix, 'application/octet-stream')) + else: + req.fulfill(status=404) + page.route('https://hope.test/**', route) + page.goto('https://hope.test/'); page.wait_for_load_state('networkidle') + page.evaluate("window.voiceEvents=[];window.boundEvents=[];addEventListener('hope:voice-state',e=>voiceEvents.push(e.detail));addEventListener('hope:voice-conversation-bound',e=>boundEvents.push(e.detail));") + assert page.evaluate('HopeVoice.state().ready') is False + page.evaluate('HopeVoice.start()') + page.wait_for_function('HopeVoice.state().ready') + state = page.evaluate('HopeVoice.state()') + assert state['connected'] and state['agent_connected'] and state['microphone_enabled'] + assert state['conversation_id'] == 'voice-cid' + assert page.evaluate('boundEvents[0].conversation_id') == 'voice-cid' + page.evaluate('HopeVoice.mute()') + assert page.evaluate('HopeVoice.state().muted && !HopeVoice.state().microphone_enabled && !HopeVoice.state().ready') + page.evaluate('HopeVoice.mute()') + assert page.evaluate('HopeVoice.state().ready') + page.evaluate("dispatchEvent(new CustomEvent('hope:conversation-selected',{detail:{conversation_id:'other-cid'}}))") + assert page.evaluate('!HopeVoice.state().active && !HopeVoice.state().connected') + assert page.evaluate('voiceEvents.at(-1).conversation_id') == 'voice-cid' + page.evaluate('window.denyMic=true;HopeVoice.start().catch(()=>{})') + page.wait_for_function("document.getElementById('status').textContent.includes('Permission denied')") + assert page.evaluate('!HopeVoice.state().active && !HopeVoice.state().ready && !HopeVoice.state().microphone_enabled') + assert errors == [] + browser.close() diff --git a/web/app.js b/web/app.js index d3cd7e6..6a11908 100644 --- a/web/app.js +++ b/web/app.js @@ -29,6 +29,11 @@ let userSpeaking = false; // local VAD-ish flag from mic level let wakeLock = null; let conversationMode = localStorage.getItem(MODE_KEY) === "text" ? "text" : "voice"; let connectionPromise = null; +let connectionEpoch = 0; +let tokenAbort = null; +let voiceConversationId = null; +let voiceSessionActive = false; +let lastVoiceState = ""; function escapeHtml(s) { const d = document.createElement("div"); @@ -133,6 +138,7 @@ async function setConversationMode(mode) { } if (!room || !agentReady || !room.localParticipant) return; + const modeRoom = room; sendModeMessage(mode); if (mode === "text") { await room.localParticipant.setMicrophoneEnabled(false); @@ -148,7 +154,9 @@ async function setConversationMode(mode) { startVisualLoop(); try { await room.startAudio(); - await room.localParticipant.setMicrophoneEnabled(true); + await modeRoom.localParticipant.setMicrophoneEnabled(true); + if (modeRoom !== room) { await modeRoom.localParticipant.setMicrophoneEnabled(false); return; } + micMuted = !microphoneEnabled(); setupMicAnalyser(); statusEl.textContent = "Secure room · Microphone on"; } catch (err) { @@ -160,6 +168,7 @@ async function setConversationMode(mode) { statusEl.textContent = "Voice mode · Microphone unavailable"; } } + updateMuteIcon(); setState(uiState); } voiceModeBtn.addEventListener("click", () => setConversationMode("voice").catch(console.error)); @@ -243,6 +252,8 @@ function buildLedWall() { // Renders one frame of the LED wall. `level` is 0..1 (agent audio when speaking, // mic when listening). `mode` picks the palette/gradient by conversation state. function renderLedWall(level, mode) { + const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches; + if (reducedMotion) { level = 0; mode = "idle"; } ledWallTime += 1; ledWallEl.dataset.mode = mode; const n = ledDots.length; @@ -279,8 +290,8 @@ function renderLedWall(level, mode) { vis = Math.max(0.02, Math.min(1, vis)); // Dot grows toward the center and sits on a subtle 4px grid inset. - const px = (2 * size).toFixed(1); - dot.style.setProperty("--sz", `${px}px`); + + dot.style.transform = `scale(${size.toFixed(2)})`; dot.style.opacity = vis.toFixed(3); } } @@ -330,9 +341,13 @@ const STATE_LABELS = { }; function setState(next) { + if (["listening", "speaking"].includes(next) && conversationMode === "voice" && !microphoneEnabled()) next = voiceSessionActive ? "connecting" : "idle"; uiState = next; ledWallEl.dataset.state = next; - stateLabelEl.textContent = STATE_LABELS[conversationMode][next] || next; + publishVoiceState(); + stateLabelEl.textContent = conversationMode === "voice" && voiceSessionActive && agentReady && micMuted + ? "Microphone muted" + : STATE_LABELS[conversationMode][next] || next; // Thinking timer: show the "thinking..." chip if it lingers past 2s if (next === "thinking") { @@ -358,7 +373,7 @@ updateModeUI(); function inferState() { if (!room || !room.localParticipant) return "idle"; - if (!agentParticipant) return "connecting"; + if (!agentParticipant || !agentReady || !microphoneEnabled()) return "connecting"; readMicLevel(); readAgentAudioLevel(); userSpeaking = micLevel > 0.035; @@ -565,6 +580,7 @@ function handleDataPacket(payload, participant, topic) { if (!HopeAuth.user || !participant || !isAgent(participant)) return; if (msg.type === "agent_ready" && topic === "conversation-control") { agentReady = true; + publishVoiceState(); sendVoiceMessage(currentVoiceId()); } else if (msg.type === "transcript" && topic === "transcript") { if (!["agent", "user"].includes(msg.role) || typeof msg.text !== "string") return; @@ -833,6 +849,7 @@ function openSheet() { if (!HopeAuth.user) return; renderVoiceList(); loadVoiceCatalog(); + if (window.HopeSettings) { window.HopeSettings.open("preferences"); return; } settingsSheet.hidden = false; settingsSheet.inert = false; document.getElementById("authenticatedApp").inert = true; sheetBackdrop.hidden = false; @@ -844,6 +861,7 @@ function openSheet() { } function closeSheet() { + if (window.HopeSettings) { window.HopeSettings.close(); return; } settingsSheet.classList.remove("open"); settingsSheet.hidden = true; settingsSheet.inert = true; document.getElementById("authenticatedApp").inert = false; @@ -914,6 +932,7 @@ function findAgent(rm) { function markAgentFound(rm, participant) { if (rm !== room || !HopeAuth.user) return; agentParticipant = participant; + publishVoiceState(); statusEl.textContent = conversationMode === "text" ? "Hope joined · Text chat ready" : "Hope joined · Preparing microphone…"; @@ -934,6 +953,14 @@ async function createRoom(token, attempt) { if (newRoom === room && isAgent(participant)) markAgentFound(newRoom, participant); }); + newRoom.on(RoomEvent.ParticipantDisconnected, participant => { + if (newRoom === room && participant === agentParticipant) { + stopConversation(); statusEl.textContent = "Hope disconnected. Start again to reconnect."; + } + }); + newRoom.on(RoomEvent.TrackMuted, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); publishVoiceState(); } }); + newRoom.on(RoomEvent.TrackUnmuted, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); publishVoiceState(); } }); + newRoom.on(RoomEvent.LocalTrackUnpublished, () => { if (newRoom === room) { micMuted = !microphoneEnabled(); updateMuteIcon(); } }); // Attach and play incoming audio tracks (agent TTS) newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => { if (newRoom === room && isAgent(participant) && track.kind === LivekitClient.Track.Kind.Audio) { @@ -992,16 +1019,33 @@ async function createRoom(token, attempt) { }); await newRoom.connect(LIVEKIT_URL, token, { autoSubscribe: true }); checkAttempt(attempt); + publishVoiceState(); const existing = findAgent(newRoom); if (existing) markAgentFound(newRoom, existing); return newRoom; } -let connectionEpoch = 0; -let tokenAbort = null; -let voiceConversationId = null; -function publishVoiceState(active) { - window.dispatchEvent(new CustomEvent('hope:voice-state', { detail: { active, conversation_id: voiceConversationId } })); +function microphoneEnabled() { + const local = room?.localParticipant; + if (!local || !local.isMicrophoneEnabled) return false; + const track = local.getTrackPublication?.(LivekitClient.Track.Source.Microphone)?.track?.mediaStreamTrack; + return !track || (track.enabled && track.readyState !== "ended"); +} +function voiceState() { + const connected = !!room && room.state === "connected"; + const microphone_enabled = connected && microphoneEnabled(); + const agent_connected = connected && !!agentParticipant; + const ready = agent_connected && agentReady && microphone_enabled && !micMuted; + return { active: voiceSessionActive, state: voiceSessionActive ? (ready ? uiState : (agent_connected && agentReady && micMuted ? "muted" : "connecting")) : "idle", + ready, muted: micMuted, connected, agent_connected, microphone_enabled, + conversation_id: voiceConversationId, cid: voiceConversationId }; +} +function publishVoiceState() { + const detail = voiceState(); + const signature = JSON.stringify(detail); + if (signature === lastVoiceState) return; + lastVoiceState = signature; + window.dispatchEvent(new CustomEvent('hope:voice-state', { detail })); } window.addEventListener('hope:conversation-selected', event => { if ((room || connectionPromise) && event.detail?.conversation_id !== voiceConversationId) stopConversation(); @@ -1028,7 +1072,7 @@ function cleanupRoom() { document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove()); document.getElementById("audioUnlockBtn")?.remove(); muteBtn.disabled = true; micMuted = false; updateMuteIcon(); - publishVoiceState(false); voiceConversationId = null; + voiceSessionActive = false; publishVoiceState(); } function stopConversation() { connectionEpoch++; tokenAbort?.abort(); tokenAbort = null; connectionPromise = null; @@ -1044,7 +1088,7 @@ async function connectConversation() { const attempt = ++connectionEpoch; if (window.HopeChat?.running) throw new Error('Wait for the current response or stop it before starting voice.'); voiceConversationId = window.HopeChat?.conversationId || null; - publishVoiceState(true); + voiceSessionActive = true; publishVoiceState(); startBtn.disabled = true; stopBtn.disabled = false; const promise = (async () => { try { @@ -1079,9 +1123,16 @@ async function connectConversation() { return promise; } -startBtn.addEventListener("click", () => { - connectConversation().catch(() => {}); -}); +async function startVoice() { + if (window.HopeChat?.running) throw new Error("Wait for the current response or stop it before starting voice."); + await setConversationMode("voice"); + return connectConversation(); +} +window.HopeVoiceSettings = Object.freeze({ load: () => { renderVoiceList(); return Promise.all([loadVoiceCatalog(), loadModelSettings()]); } }); +window.HopeVoice = Object.freeze({ start: startVoice, stop: stopConversation, mute: toggleMute, state: voiceState, + loadPreferences: () => Promise.all([loadVoiceCatalog(), loadModelSettings()]) }); +window.addEventListener("hope:start-voice", () => startVoice().catch(error => { statusEl.textContent = error.message; })); +startBtn.addEventListener("click", () => startVoice().catch(error => { statusEl.textContent = error.message; })); // ── Text chat ────────────────────────────────────────────────────────────── function resizeChatInput() { @@ -1130,20 +1181,25 @@ chatComposer.addEventListener("submit", async (event) => { stopBtn.addEventListener("click", stopConversation); // ── Mic mute toggle ───────────────────────────────────────────────────────── -muteBtn.addEventListener("click", async () => { - if (!room || !room.localParticipant) return; - const nextMuted = !micMuted; +async function toggleMute() { + if (!room || !agentReady || !room.localParticipant || conversationMode !== "voice") return; + const active = room; + const nextMuted = microphoneEnabled(); try { - await room.localParticipant.setMicrophoneEnabled(!nextMuted); - micMuted = nextMuted; + await active.localParticipant.setMicrophoneEnabled(!nextMuted); + if (active !== room) return; + micMuted = !microphoneEnabled(); updateMuteIcon(); } catch (err) { console.warn("Microphone toggle failed:", err); + micMuted = !microphoneEnabled(); updateMuteIcon(); statusEl.textContent = "Microphone unavailable in this browser"; } -}); +} +muteBtn.addEventListener("click", toggleMute); function updateMuteIcon() { + publishVoiceState(); const muted = muteBtn.classList.toggle("muted", micMuted); muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone"); muteBtn.title = muted ? "Unmute microphone" : "Mute microphone"; @@ -1156,7 +1212,8 @@ async function fetchToken(attempt) { const data = await HopeAuth.request("/token", { method: "POST", body: JSON.stringify(voiceConversationId ? { conversation_id: voiceConversationId } : {}), signal: tokenAbort.signal }); checkAttempt(attempt); voiceConversationId = data.conversation_id || voiceConversationId; - publishVoiceState(true); + if (voiceConversationId) window.dispatchEvent(new CustomEvent('hope:voice-conversation-bound', { detail: { conversation_id: voiceConversationId } })); + publishVoiceState(); if (!data.token || typeof data.token !== "string") throw new Error("Token endpoint did not return a room token"); return data; } @@ -1183,7 +1240,7 @@ function sendModeMessage(mode, targetRoom) { } HopeAuth.onChange((user, changed) => { - if (changed || !user || user.status !== "ready") stopConversation(); + if (changed || !user || user.status !== "ready") { stopConversation(); voiceConversationId = null; publishVoiceState(); } if (changed || settingsAccount !== (user ? String(user.id) : null)) resetModelSettings(user); if (changed) { voiceCatalogGeneration++;