// Voice Assistant — LiveKit client const { Room, RoomEvent } = LivekitClient; // ── Config ────────────────────────────────────────────────────────────────── // LiveKit is reached through the same origin as the page (nginx proxies // /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed // content when the UI is served over HTTPS. const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`; const AGENT_NAME = "voice-assistant"; const OMNI_DEFAULT = "en-US-Ava:DragonHDOmniLatestNeural"; let defaultVoice = OMNI_DEFAULT; let VOICES = [{ id: OMNI_DEFAULT, name: "Ava", locale: "en-US", gender: "Female", tag: "Dragon HD Omni" }]; let voiceCatalogRequest = null; let voiceCatalogGeneration = 0; let historyKey = null; const HISTORY_MAX = 50; const MODE_KEY = "hope_conversation_mode"; // ── State ─────────────────────────────────────────────────────────────────── let room = null; let agentParticipant = null; let agentReady = false; let uiState = "idle"; // idle | connecting | listening | thinking | speaking let micLevel = 0; // 0..1, smoothed let agentAudioLevel = 0; // 0..1, smoothed let userSpeaking = false; // local VAD-ish flag from mic level let wakeLock = null; let conversationMode = localStorage.getItem(MODE_KEY) === "text" ? "text" : "voice"; let connectionPromise = null; function escapeHtml(s) { const d = document.createElement("div"); d.textContent = s || ""; return d.innerHTML; } // ── DOM ───────────────────────────────────────────────────────────────────── const startBtn = document.getElementById("startBtn"); const stopBtn = document.getElementById("stopBtn"); const muteBtn = document.getElementById("muteBtn"); let micMuted = false; const statusEl = document.getElementById("status"); const messagesEl = document.getElementById("messages"); const ledWallEl = document.getElementById("ledWall"); const stateLabelEl = document.getElementById("stateLabel"); const thinkingChipEl = document.getElementById("thinkingChip"); const bargeHintEl = document.getElementById("bargeHint"); const micMeterFillEl = document.getElementById("micMeterFill"); const settingsBtn = document.getElementById("settingsBtn"); const settingsSheet = document.getElementById("settingsSheet"); const sheetBackdrop = document.getElementById("sheetBackdrop"); const closeSheetBtn = document.getElementById("closeSheetBtn"); const voiceListEl = document.getElementById("voiceList"); const clearBtn = document.getElementById("clearBtn"); const tasksBtn = document.getElementById("tasksBtn"); const tasksPanel = document.getElementById("tasksPanel"); const tasksList = document.getElementById("tasksList"); const tasksBadge = document.getElementById("tasksBadge"); const closeTasksBtn = document.getElementById("closeTasksBtn"); const voiceModeBtn = document.getElementById("voiceModeBtn"); const textModeBtn = document.getElementById("textModeBtn"); const modeDescriptionEl = document.getElementById("modeDescription"); const transcriptModeLabelEl = document.getElementById("transcriptModeLabel"); const chatComposer = document.getElementById("chatComposer"); const chatInput = document.getElementById("chatInput"); const sendChatBtn = document.getElementById("sendChatBtn"); // Activity is progress only, never a second assistant transcript. const progress = new Map(); function handleProgress(data) { if (typeof data.request_id !== "string" || !["working", "completed", "failed"].includes(data.status) || typeof data.message !== "string") return; progress.set(data.request_id, { ...data, updated: Date.now() }); while (progress.size > 50) progress.delete(progress.keys().next().value); const working = [...progress.values()].filter(item => item.status === "working").length; tasksBadge.hidden = !working; tasksBadge.textContent = String(working); tasksList.replaceChildren(); for (const item of [...progress.values()].sort((a, b) => b.updated - a.updated)) { const row = document.createElement("div"); row.className = "task-card"; const state = document.createElement("strong"); state.textContent = { working: "Working", completed: "Completed", failed: "Failed" }[item.status]; const text = document.createElement("p"); text.textContent = item.message; row.append(state, text); tasksList.append(row); } statusEl.textContent = data.message || { working: "Hope is working…", completed: "Hope finished", failed: "Hope could not complete this request" }[data.status]; setState(working ? "thinking" : "listening"); } function setTasksOpen(open) { tasksPanel.hidden = !open; tasksPanel.inert = !open; tasksPanel.classList.toggle("open", open); tasksBtn.setAttribute("aria-expanded", String(open)); if (open) closeTasksBtn.focus(); else tasksBtn.focus(); } tasksBtn.setAttribute("aria-expanded", "false"); tasksBtn.addEventListener("click", () => setTasksOpen(!tasksPanel.classList.contains("open"))); closeTasksBtn.addEventListener("click", () => setTasksOpen(false)); // ── Voice / text mode ────────────────────────────────────────────────────── function updateModeUI() { const isText = conversationMode === "text"; document.body.dataset.mode = conversationMode; voiceModeBtn.classList.toggle("active", !isText); textModeBtn.classList.toggle("active", isText); voiceModeBtn.setAttribute("aria-pressed", String(!isText)); textModeBtn.setAttribute("aria-pressed", String(isText)); modeDescriptionEl.textContent = isText ? "Type when quiet is better. Switch back to voice without losing the thread." : "Speak naturally. I’ll listen, remember, and answer in real time."; transcriptModeLabelEl.textContent = isText ? "Shared voice and text history" : "Live voice transcript"; startBtn.innerHTML = isText ? 'Connect to Hope' : 'Start conversation'; if (isText) { setState(room ? "listening" : "idle"); window.setTimeout(() => chatInput.focus(), 0); } else { // Refresh mode-specific copy even when the underlying state is unchanged. setState(uiState); } } async function setConversationMode(mode) { if (mode !== "voice" && mode !== "text") return; const previousMode = conversationMode; conversationMode = mode; localStorage.setItem(MODE_KEY, mode); updateModeUI(); if (mode === "voice" && previousMode === "text") { chatInput.blur(); window.setTimeout(() => { document.querySelector(".workspace").scrollIntoView({ block: "start", behavior: "smooth" }); }, 0); } if (!room || !agentReady || !room.localParticipant) return; sendModeMessage(mode); if (mode === "text") { await room.localParticipant.setMicrophoneEnabled(false); micMuted = true; muteBtn.disabled = true; updateMuteIcon(); stopVisualLoop(); statusEl.textContent = "Secure room · Text mode"; } else { micMuted = false; muteBtn.disabled = false; updateMuteIcon(); startVisualLoop(); try { await room.startAudio(); await room.localParticipant.setMicrophoneEnabled(true); setupMicAnalyser(); statusEl.textContent = "Secure room · Microphone on"; } catch (err) { // Keep the conversation usable even if this browser denies or does // not implement microphone capture. The mute control can retry. console.warn("Microphone could not be enabled:", err); micMuted = true; updateMuteIcon(); statusEl.textContent = "Voice mode · Microphone unavailable"; } } } voiceModeBtn.addEventListener("click", () => setConversationMode("voice").catch(console.error)); textModeBtn.addEventListener("click", () => setConversationMode("text").catch(console.error)); // ── Audio analysers (mic + remote agent audio) ───────────────────────────── let micCtx = null; let micAnalyser = null; let micBuf = null; let remoteCtx = null; let remoteAnalyser = null; let remoteFreqBuf = null; function setupMicAnalyser() { try { const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone); const mst = pub && pub.track && pub.track.mediaStreamTrack; if (mst) { micCtx = new AudioContext(); micAnalyser = micCtx.createAnalyser(); micAnalyser.fftSize = 512; micBuf = new Float32Array(micAnalyser.fftSize); micCtx.createMediaStreamSource(new MediaStream([mst])).connect(micAnalyser); } } catch (e) { /* fall back to SDK audioLevel in the meter loop */ } } function setupRemoteAudioAnalyser(audioEl) { try { if (!audioEl.srcObject) return; remoteCtx = new AudioContext(); remoteAnalyser = remoteCtx.createAnalyser(); remoteAnalyser.fftSize = 256; remoteFreqBuf = new Uint8Array(remoteAnalyser.frequencyBinCount); remoteCtx.createMediaStreamSource(audioEl.srcObject).connect(remoteAnalyser); } catch (e) { /* analyser optional */ } } function teardownAudio() { for (const ctx of [micCtx, remoteCtx]) { if (ctx && ctx.state !== "closed") ctx.close().catch(() => {}); } micCtx = micAnalyser = micBuf = null; remoteCtx = remoteAnalyser = remoteFreqBuf = null; } // ── LED wall — a grid of dots that pulses with the agent's audio ──────────── const LED_COLS = 21; const LED_ROWS = 21; const LED_GROW = 1.4; // how much the dots grow toward the center let ledDots = []; let ledWallTime = 0; function buildLedWall() { const inner = Math.floor(LED_COLS / 2); const frag = document.createDocumentFragment(); for (let r = 0; r < LED_ROWS; r++) { for (let c = 0; c < LED_COLS; c++) { const b = document.createElement("button"); b.type = "button"; b.className = "led-dot"; b.disabled = true; b.tabIndex = -1; // Radial position: 0 at the center, 1 at the corner const dx = (c - inner) / (LED_COLS / 2); const dy = (r - inner) / (LED_ROWS / 2); const rad = Math.sqrt(dx * dx + dy * dy); // Push a radial cluster toward the center; scale its base brightness down // as it moves outward so the wall reads as a soft round bloom. b.dataset.rad = rad.toFixed(4); // A per-dot phase so the wave ripples rather than flashing in unison. b.dataset.phase = (Math.sin(r * 12.9898 + c * 78.233) * 43758.5453 % 1 * Math.PI * 2).toFixed(3); ledDots.push(b); frag.appendChild(b); } } ledWallEl.appendChild(frag); } // Renders one frame of the LED wall. `level` is 0..1 (agent audio when speaking, // mic when listening). `mode` picks the palette/gradient by conversation state. function renderLedWall(level, mode) { ledWallTime += 1; ledWallEl.dataset.mode = mode; const n = ledDots.length; for (let i = 0; i < n; i++) { const dot = ledDots[i]; const rad = parseFloat(dot.dataset.rad); const phase = parseFloat(dot.dataset.phase || 0); let vis = 0; let size = 1; if (mode === "idle") { // A faint, static dot matrix — present but clearly dormant. vis = 0.16 - rad * 0.08; size = 1; } else if (level > 0.005) { // Centered breathing bloom that widens and brightens with the // signal. A soft gaussian falloff keeps it circular; a gentle // per-dot phase ripple adds life without banding. const lvl = Math.min(1, level * 1.3); const falloff = Math.exp(-rad * rad * (2.0 - lvl * 1.1)); const ripple = 0.62 + 0.38 * Math.sin(ledWallTime * 0.32 + phase); vis = falloff * (0.35 + lvl) * ripple; size = 1 + (1 - rad) * LED_GROW * lvl; // Keep listening dimmer than speaking so the wall leans into the // agent's output. if (mode !== "speaking") vis *= 0.55; } else { // Powered state with no signal (e.g. thinking gap) — soft breathing. const breathe = 0.5 + 0.5 * Math.sin(ledWallTime * 0.12 + phase); vis = 0.14 - rad * 0.06 + 0.04 * breathe; size = 1 + (1 - rad) * 0.3; } vis = Math.max(0.02, Math.min(1, vis)); // Dot grows toward the center and sits on a subtle 4px grid inset. const px = (2 * size).toFixed(1); dot.style.setProperty("--sz", `${px}px`); dot.style.opacity = vis.toFixed(3); } } function readMicLevel() { let level = 0; if (micAnalyser && micBuf) { micAnalyser.getFloatTimeDomainData(micBuf); for (let i = 0; i < micBuf.length; i++) level = Math.max(level, Math.abs(micBuf[i])); } else if (room && room.localParticipant) { level = room.localParticipant.audioLevel || 0; } // Smooth so the orb pulses rather than flickers micLevel = micLevel * 0.6 + level * 0.4; return micLevel; } function readAgentAudioLevel() { let level = 0; if (remoteAnalyser && remoteFreqBuf) { remoteAnalyser.getByteFrequencyData(remoteFreqBuf); for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]); level /= 255; } else if (agentParticipant && agentParticipant.audioLevel !== undefined) { level = agentParticipant.audioLevel || 0; } agentAudioLevel = agentAudioLevel * 0.6 + level * 0.4; return agentAudioLevel; } // ── State machine ─────────────────────────────────────────────────────────── const STATE_LABELS = { voice: { idle: "Ready when you are", connecting: "Opening a secure room", listening: "Listening", thinking: "Thinking it through", speaking: "Hope is speaking", }, text: { idle: "Ready to type", connecting: "Opening a secure room", listening: "Text chat ready", thinking: "Hope is writing", speaking: "Hope replied", }, }; function setState(next) { uiState = next; ledWallEl.dataset.state = next; stateLabelEl.textContent = STATE_LABELS[conversationMode][next] || next; // Thinking timer: show the "thinking..." chip if it lingers past 2s if (next === "thinking") { thinkingChipEl.hidden = true; setTimeout(() => { if (uiState === "thinking") thinkingChipEl.hidden = false; }, 2000); } else { thinkingChipEl.hidden = true; } // Barge-in hint only while the agent is speaking bargeHintEl.hidden = next !== "speaking"; if (next === "idle") { micLevel = 0; agentAudioLevel = 0; userSpeaking = false; } } updateModeUI(); function inferState() { if (!room || !room.localParticipant) return "idle"; if (!agentParticipant) return "connecting"; readMicLevel(); readAgentAudioLevel(); userSpeaking = micLevel > 0.035; if (agentAudioLevel > 0.02) return "speaking"; if (userSpeaking) return "listening"; // Once agent audio ends, return to listening. The previous implementation // converted speaking → thinking forever, leaving the UI apparently frozen. if (uiState === "speaking") return "listening"; if (uiState === "thinking") return "thinking"; return "listening"; } // ── Visual loop: drives orb + meter + state inference ─────────────────────── let visualTimer = null; function startVisualLoop() { if (visualTimer) clearInterval(visualTimer); visualTimer = setInterval(() => { if (!room || !room.localParticipant) return; setState(inferState()); // Mic meter (thin bar under the status line) micMeterFillEl.style.width = `${Math.min(100, Math.round(micLevel * 250))}%`; // LED wall: pulse on the agent's rendered audio when speaking, the mic // while listening, and hold a faint matrix when idle. if (uiState === "speaking") { renderLedWall(agentAudioLevel, "speaking"); } else if (uiState === "listening" && userSpeaking) { renderLedWall(micLevel, "listening"); } else { renderLedWall(0, "idle"); } // Agent audio visualizer bars on the latest agent bubble const viz = messagesEl.querySelector(".message-row.agent:last-of-type .audio-viz"); if (viz) { let level = 0; if (remoteAnalyser && remoteFreqBuf) { for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]); level /= 255; } const bars = viz.children; for (let i = 0; i < bars.length; i++) { // Per-bar phase offset so the bars dance rather than move in lockstep const h = Math.max(0.15, level * (0.6 + 0.4 * Math.sin(Date.now() / 120 + i * 1.3))); bars[i].style.height = `${h * 100}%`; } } }, 80); } function stopVisualLoop() { if (visualTimer) clearInterval(visualTimer); visualTimer = null; micMeterFillEl.style.width = "0%"; renderLedWall(0, "idle"); } // ── Conversation history (localStorage) ───────────────────────────────────── function loadHistory() { try { if (!historyKey) return []; const raw = localStorage.getItem(historyKey); if (!raw) return []; const arr = JSON.parse(raw); return Array.isArray(arr) ? arr.filter(msg => msg && ["user", "agent", "system"].includes(msg.role) && typeof msg.text === "string").slice(-HISTORY_MAX) : []; } catch (e) { return []; } } function persistHistory() { try { if (!historyKey) return; localStorage.setItem(historyKey, JSON.stringify(history.slice(-HISTORY_MAX))); } catch (e) { /* storage full or unavailable */ } } let history = []; const DELIVERY_MARKERS = /\[(?:appreciative|calm|concerned|curious|encouraging|excited|happy|joking|quiet|reassuring|reflective|sad|serious|surprised|sympathetic|urgent|whispering|laughter|sighing|breathing|neutral)\]\s*/gi; function cleanSpokenText(text) { return String(text || "").replace(DELIVERY_MARKERS, "").trim(); } function formatTimestamp(ts) { if (!ts) return ""; return new Date(ts).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }); } function renderMessage(msg) { const wrap = document.createElement("div"); wrap.className = `message-row ${msg.role}`; wrap.dataset.ts = formatTimestamp(msg.ts); if (msg.role === "agent") { const dot = document.createElement("span"); dot.className = "avatar-dot"; wrap.appendChild(dot); } const bubble = document.createElement("div"); bubble.className = "bubble"; bubble.textContent = cleanSpokenText(msg.text); if (msg.role === "agent") { const viz = document.createElement("span"); viz.className = "audio-viz"; for (let i = 0; i < 5; i++) viz.appendChild(document.createElement("i")); bubble.appendChild(viz); } wrap.appendChild(bubble); messagesEl.appendChild(wrap); messagesEl.scrollTop = messagesEl.scrollHeight; } function addMessage(role, text) { clearPartials(); const msg = { role, text, ts: Date.now() }; history.push(msg); persistHistory(); renderMessage(msg); return msg; } // Partial transcripts render greyed and update in place until a final arrives. function addPartial(role, text) { const rows = messagesEl.querySelectorAll(".message-row.partial"); let last = null; for (const r of rows) last = r; if (last && last.dataset.role === role) { last.querySelector(".bubble").textContent = cleanSpokenText(text); messagesEl.scrollTop = messagesEl.scrollHeight; return; } const wrap = document.createElement("div"); wrap.className = `message-row ${role} partial`; wrap.dataset.role = role; if (role === "agent") { const dot = document.createElement("span"); dot.className = "avatar-dot"; wrap.appendChild(dot); } const bubble = document.createElement("div"); bubble.className = "bubble partial"; bubble.textContent = cleanSpokenText(text); wrap.appendChild(bubble); messagesEl.appendChild(wrap); messagesEl.scrollTop = messagesEl.scrollHeight; } function clearPartials() { messagesEl.querySelectorAll(".message-row.partial").forEach((el) => el.remove()); } function renderHistory() { messagesEl.innerHTML = ""; for (const msg of history) renderMessage(msg); messagesEl.scrollTop = messagesEl.scrollHeight; } clearBtn.addEventListener("click", () => { history = []; persistHistory(); clearPartials(); renderHistory(); }); renderHistory(); // ── Transcripts ───────────────────────────────────────────────────────────── // Primary: livekit-agents built-in text stream (word-by-word when available). function registerTranscriptionStream(rm) { try { rm.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => { const text = await reader.readAll(); if (rm !== room || !HopeAuth.user || !text || !text.trim()) return; const sender = rm.remoteParticipants.get(participantInfo?.identity); if (!sender || !isAgent(sender)) return; const attrs = (reader.info && reader.info.attributes) || {}; const trackId = attrs["lk.transcribed_track_id"] || ""; // Distinguish user vs agent by the transcribed track's owner. let role = "agent"; if (participantInfo && participantInfo.identity === AGENT_NAME) { role = "agent"; } else if (trackId) { for (const p of rm.remoteParticipants.values()) { const pub = p.getTrackPublication(trackId); if (pub) { role = isAgent(p) ? "agent" : "user"; break; } } } // Use partial rendering: updates the same row in place until final. addPartial(role, text.trim()); }); } catch (e) { // Older SDKs may not support registerTextStreamHandler — the data // channel handler below still covers transcripts. console.warn("text stream registration failed:", e); } } // Secondary: agent publishes {type:"transcript"} on topic "transcript". function handleDataPacket(payload, participant, topic) { try { const msg = JSON.parse(new TextDecoder().decode(payload)); if (!HopeAuth.user || !participant || !isAgent(participant)) return; if (msg.type === "agent_ready" && topic === "conversation-control") { agentReady = true; sendVoiceMessage(currentVoiceId()); } else if (msg.type === "transcript" && topic === "transcript") { if (!["agent", "user"].includes(msg.role) || typeof msg.text !== "string") return; addMessage(msg.role, msg.text); if (msg.role === "agent") setState(conversationMode === "text" ? "listening" : "speaking"); } else if (msg.type === "assistant_progress" && topic === "assistant-progress") { handleProgress(msg); } } catch (e) { console.warn("bad data packet", e); } } // ── Account model settings (server saved; never browser persisted) ────────── const modelSettingsForm = document.getElementById("modelSettingsForm"); const modelSelect = document.getElementById("modelSelect"); const thinkingLevelSelect = document.getElementById("thinkingLevelSelect"); const saveModelSettingsBtn = document.getElementById("saveModelSettings"); const retryModelSettingsBtn = document.getElementById("retryModelSettings"); const modelSettingsStatus = document.getElementById("modelSettingsStatus"); const thinkingLevelNote = document.getElementById("thinkingLevelNote"); const THINKING_LEVELS = [ ["none", "Off"], ["minimal", "Minimal"], ["low", "Low"], ["medium", "Medium"], ["high", "High"], ["xhigh", "Extra high"], ]; const THINKING_NOTE = "Thinking requests may be limited or ignored by the selected model."; let settingsAccount = null; let settingsGeneration = 0; let settingsAbort = null; let settingsBusy = false; let modelCatalog = []; let catalogAvailable = false; let savedModelSettings = null; function accountIdentity() { return HopeAuth.user ? String(HopeAuth.user.id) : null; } function settingsResponseIsCurrent(account, generation) { return account !== null && account === accountIdentity() && account === settingsAccount && generation === settingsGeneration; } function settingsOption(value, label) { const option = document.createElement("option"); option.value = value; option.textContent = label; return option; } function setSettingsStatus(message, state = "") { modelSettingsStatus.textContent = message; modelSettingsStatus.dataset.state = state; } function updateSettingsControls() { const available = !!HopeAuth.user && catalogAvailable && !settingsBusy; modelSelect.disabled = !available; thinkingLevelSelect.disabled = !available; const model = modelCatalog.find(item => item.id === modelSelect.value); const valid = model && model.thinking_levels.includes(thinkingLevelSelect.value); const changed = savedModelSettings && (modelSelect.value !== savedModelSettings.model || thinkingLevelSelect.value !== savedModelSettings.thinking_level); saveModelSettingsBtn.disabled = !available || !valid || !changed; retryModelSettingsBtn.disabled = settingsBusy; saveModelSettingsBtn.textContent = settingsBusy && modelSettingsForm.dataset.operation === "save" ? "Saving…" : "Save"; modelSettingsForm.setAttribute("aria-busy", String(settingsBusy)); } function renderThinkingLevels(preferred) { const model = modelCatalog.find(item => item.id === modelSelect.value); const levels = model?.thinking_levels || []; thinkingLevelSelect.replaceChildren(); for (const [value, label] of THINKING_LEVELS) { if (levels.includes(value) || value === preferred) thinkingLevelSelect.append(settingsOption(value, label)); } if (!thinkingLevelSelect.options.length) thinkingLevelSelect.append(settingsOption("", "No thinking levels available")); if ([...thinkingLevelSelect.options].some(option => option.value === preferred)) thinkingLevelSelect.value = preferred; updateSettingsControls(); } function renderSavedModelSettings() { modelSelect.replaceChildren(); for (const model of modelCatalog) modelSelect.append(settingsOption(model.id, model.name || model.id)); if (savedModelSettings && !modelCatalog.some(model => model.id === savedModelSettings.model)) { modelSelect.prepend(settingsOption(savedModelSettings.model, `${savedModelSettings.model} (saved; unavailable)`)); } if (!modelSelect.options.length) modelSelect.append(settingsOption("", "Model catalog unavailable")); if (savedModelSettings) modelSelect.value = savedModelSettings.model; renderThinkingLevels(savedModelSettings?.thinking_level); } function resetModelSettings(user) { settingsGeneration++; settingsAbort?.abort(); settingsAbort = null; settingsAccount = user ? String(user.id) : null; settingsBusy = false; catalogAvailable = false; modelCatalog = []; savedModelSettings = null; modelSettingsForm.dataset.operation = ""; modelSelect.replaceChildren(settingsOption("", "Open settings to load models")); thinkingLevelSelect.replaceChildren(settingsOption("", "Open settings to load thinking levels")); thinkingLevelNote.textContent = THINKING_NOTE; retryModelSettingsBtn.hidden = true; setSettingsStatus("Open settings to load your saved preferences."); updateSettingsControls(); } function readSavedSettings(data) { if (!data?.settings || typeof data.settings.model !== "string" || !THINKING_LEVELS.some(([level]) => level === data.settings.thinking_level)) { throw new Error("The server did not return valid saved settings."); } return { model: data.settings.model, thinking_level: data.settings.thinking_level }; } async function loadModelSettings() { const account = accountIdentity(); if (!account || settingsBusy) return; if (settingsAccount !== account) resetModelSettings(HopeAuth.user); const generation = ++settingsGeneration; settingsAbort?.abort(); settingsAbort = new AbortController(); settingsBusy = true; modelSettingsForm.dataset.operation = "load"; retryModelSettingsBtn.hidden = true; setSettingsStatus("Loading your saved model settings…"); updateSettingsControls(); try { const data = await HopeAuth.request("/api/settings/model", { signal: settingsAbort.signal }); if (!settingsResponseIsCurrent(account, generation)) return; savedModelSettings = readSavedSettings(data); modelCatalog = Array.isArray(data.models) ? data.models.filter(model => typeof model?.id === "string" && Array.isArray(model.thinking_levels)).map(model => ({ id: model.id, name: typeof model.name === "string" ? model.name : model.id, thinking_levels: model.thinking_levels.filter(level => THINKING_LEVELS.some(([value]) => value === level)), })) : []; catalogAvailable = data.catalog_available === true && modelCatalog.length > 0; thinkingLevelNote.textContent = typeof data.thinking_note === "string" && data.thinking_note ? data.thinking_note : THINKING_NOTE; renderSavedModelSettings(); retryModelSettingsBtn.hidden = catalogAvailable; setSettingsStatus(catalogAvailable ? (data.message || "Your saved settings are loaded. Choose preferences, then Save.") : `${data.message || "Model catalog unavailable."} Your saved preferences are shown; retry to make changes.`, catalogAvailable ? "" : "error"); } catch (error) { if (!settingsResponseIsCurrent(account, generation)) return; catalogAvailable = false; retryModelSettingsBtn.hidden = false; // Keep this account's last saved display when a refresh fails. renderSavedModelSettings(); setSettingsStatus(`Could not load model settings: ${error.message} Retry to load your saved preferences.`, "error"); } finally { if (settingsResponseIsCurrent(account, generation)) { settingsBusy = false; settingsAbort = null; updateSettingsControls(); } } } modelSelect.addEventListener("change", () => { const previous = thinkingLevelSelect.value; const model = modelCatalog.find(item => item.id === modelSelect.value); renderThinkingLevels(model?.thinking_levels.includes(previous) ? previous : model?.thinking_levels[0]); setSettingsStatus("Unsaved changes. Save to apply them to your account."); }); thinkingLevelSelect.addEventListener("change", () => { updateSettingsControls(); setSettingsStatus("Unsaved changes. Save to apply them to your account."); }); retryModelSettingsBtn.addEventListener("click", loadModelSettings); modelSettingsForm.addEventListener("submit", async event => { event.preventDefault(); updateSettingsControls(); if (saveModelSettingsBtn.disabled) return; const account = accountIdentity(), generation = ++settingsGeneration; const settings = { model: modelSelect.value, thinking_level: thinkingLevelSelect.value }; settingsAbort = new AbortController(); settingsBusy = true; modelSettingsForm.dataset.operation = "save"; retryModelSettingsBtn.hidden = true; setSettingsStatus("Saving your model settings…"); updateSettingsControls(); try { const data = await HopeAuth.request("/api/settings/model", { method: "PUT", body: JSON.stringify(settings), signal: settingsAbort.signal }); if (!settingsResponseIsCurrent(account, generation)) return; savedModelSettings = readSavedSettings(data); renderSavedModelSettings(); setSettingsStatus(data.message || "Saved. Your next request will use these account settings; your conversation continues uninterrupted.", "success"); } catch (error) { if (!settingsResponseIsCurrent(account, generation)) return; retryModelSettingsBtn.hidden = false; setSettingsStatus(`Could not save settings: ${error.message} Your changes are not confirmed saved. Try Save again, or Retry to reload saved preferences.`, "error"); } finally { if (settingsResponseIsCurrent(account, generation)) { settingsBusy = false; settingsAbort = null; updateSettingsControls(); } } }); // ── Voice settings sheet ──────────────────────────────────────────────────── function currentVoiceId() { let saved = localStorage.getItem("preferred_voice"); // Migrate older Dragon HD selections to the equivalent Omni persona. if (saved?.endsWith(":DragonHDLatestNeural")) { saved = saved.replace(":DragonHDLatestNeural", ":DragonHDOmniLatestNeural"); localStorage.setItem("preferred_voice", saved); } return VOICES.some((v) => v.id === saved) ? saved : defaultVoice; } const voiceSearchEl = document.getElementById("voiceSearch"); const voiceStatusEl = document.getElementById("voiceCatalogStatus"); const retryVoicesBtn = document.getElementById("retryVoices"); voiceSearchEl.addEventListener("input", renderVoiceList); retryVoicesBtn.addEventListener("click", loadVoiceCatalog); async function loadVoiceCatalog() { if (!HopeAuth.user) return; if (voiceCatalogRequest) return voiceCatalogRequest; const generation = voiceCatalogGeneration; voiceStatusEl.textContent = "Loading English voices from Azure…"; retryVoicesBtn.hidden = true; voiceCatalogRequest = (async () => { try { const data = await HopeAuth.request("/voices"); if (generation !== voiceCatalogGeneration) return; const voices = Array.isArray(data.voices) ? data.voices.filter(v => v && typeof v.id === "string" && /^[A-Za-z0-9:_-]{1,120}$/.test(v.id) && typeof v.name === "string") : []; if (!voices.length) throw new Error("Empty voice catalog"); VOICES = voices; defaultVoice = VOICES.some(v => v.id === data.default_voice) ? data.default_voice : OMNI_DEFAULT; voiceStatusEl.textContent = data.catalog_available ? `${VOICES.length} English voices · Dragon HD Omni first. Ava Omni is the recommended female default.` : (data.message || "Azure’s full catalog is unavailable. Ava Omni remains available; retry to load all English voices."); retryVoicesBtn.hidden = data.catalog_available === true; renderVoiceList(); } catch (_) { if (generation !== voiceCatalogGeneration) return; voiceStatusEl.textContent = "Couldn’t load Azure’s full catalog. Your current choices remain available. Retry to load all English voices."; retryVoicesBtn.hidden = false; } finally { if (generation === voiceCatalogGeneration) voiceCatalogRequest = null; } })(); return voiceCatalogRequest; } function renderVoiceList() { voiceListEl.replaceChildren(); const active = currentVoiceId(); const query = voiceSearchEl.value.trim().toLowerCase(); const visible = VOICES.filter(v => `${v.name} ${v.locale || ""} ${v.gender || ""} ${v.tag || ""}`.toLowerCase().includes(query)); if (!visible.length) { const empty = document.createElement("p"); empty.className = "model-settings-note"; empty.textContent = "No matching voices. Try a name, accent locale, gender, or model."; voiceListEl.appendChild(empty); } for (const v of visible) { const card = document.createElement("div"); card.className = "voice-card" + (v.id === active ? " active" : ""); card.dataset.voice = v.id; const name = document.createElement("span"); name.className = "voice-name"; name.textContent = `${v.name} · ${v.locale || "English"} · ${v.gender || ""}` + (v.tag ? ` (${v.tag})` : ""); const selected = v.id === active; card.setAttribute("role", "radio"); card.setAttribute("aria-checked", String(selected)); card.tabIndex = 0; card.appendChild(name); card.addEventListener("click", () => selectVoice(v.id)); card.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); selectVoice(v.id); } }); voiceListEl.appendChild(card); } } function selectVoice(voiceId) { localStorage.setItem("preferred_voice", voiceId); renderVoiceList(); if (room && agentParticipant) { sendVoiceMessage(voiceId); const v = VOICES.find((x) => x.id === voiceId); addMessage("system", `Voice changed to ${v ? v.name : voiceId}`); } } function openSheet() { if (!HopeAuth.user) return; renderVoiceList(); loadVoiceCatalog(); settingsSheet.hidden = false; settingsSheet.inert = false; document.getElementById("authenticatedApp").inert = true; sheetBackdrop.hidden = false; settingsSheet.classList.add("open"); settingsBtn.setAttribute("aria-expanded", "true"); loadModelSettings(); // The catalog is loading, so the first stable enabled control is Close. sheetFocusableControls()[0]?.focus(); } function closeSheet() { settingsSheet.classList.remove("open"); settingsSheet.hidden = true; settingsSheet.inert = true; document.getElementById("authenticatedApp").inert = false; settingsBtn.setAttribute("aria-expanded", "false"); sheetBackdrop.hidden = true; settingsBtn.focus(); } function sheetFocusableControls() { return [...settingsSheet.querySelectorAll('button, select, input, textarea, a[href], [tabindex]')].filter(el => !el.disabled && el.tabIndex >= 0 && !el.closest('[hidden], [inert]') && el.getClientRects().length > 0 && getComputedStyle(el).visibility !== "hidden" ); } settingsBtn.setAttribute("aria-expanded", "false"); settingsBtn.addEventListener("click", openSheet); closeSheetBtn.addEventListener("click", closeSheet); sheetBackdrop.addEventListener("click", closeSheet); document.addEventListener("keydown", (event) => { if (event.key === "Tab" && settingsSheet.classList.contains("open")) { const focusable = sheetFocusableControls(); const first = focusable[0], last = focusable[focusable.length - 1]; if (!focusable.includes(document.activeElement)) { event.preventDefault(); first?.focus(); } else if (event.shiftKey && document.activeElement === first) { event.preventDefault(); last?.focus(); } else if (!event.shiftKey && document.activeElement === last) { event.preventDefault(); first?.focus(); } } if (event.key !== "Escape") return; if (settingsSheet.classList.contains("open")) closeSheet(); if (tasksPanel.classList.contains("open")) setTasksOpen(false); }); // ── Wake lock ─────────────────────────────────────────────────────────────── async function acquireWakeLock() { try { if ("wakeLock" in navigator) { wakeLock = await navigator.wakeLock.request("screen"); } } catch (e) { /* not critical */ } } function releaseWakeLock() { if (wakeLock) { wakeLock.release().catch(() => {}); wakeLock = null; } } // ── Agent detection ───────────────────────────────────────────────────────── // The agent may already be in the room before we join (it persists between // page loads), so scan remoteParticipants after connect AND watch for new // joins — ParticipantConnected does not fire for pre-existing participants. function isAgent(participant) { // kind is the numeric proto enum; ParticipantKind.AGENT === 4. // Prefer the SDK's own getter where available. if (participant.isAgent !== undefined && participant.isAgent !== null) { return participant.isAgent; } return participant.kind === (LivekitClient.ParticipantKind?.AGENT ?? 4); } function findAgent(rm) { if (!rm) return null; for (const p of rm.remoteParticipants.values()) { if (isAgent(p)) return p; } return null; } function markAgentFound(rm, participant) { if (rm !== room || !HopeAuth.user) return; agentParticipant = participant; statusEl.textContent = conversationMode === "text" ? "Hope joined · Text chat ready" : "Hope joined · Preparing microphone…"; // Synchronize voice and interaction mode when the agent joins. sendVoiceMessage(currentVoiceId(), rm); sendModeMessage(conversationMode, rm); } async function createRoom(token, attempt) { const newRoom = new Room({ adaptiveStream: true, dynacast: true, }); room = newRoom; // Listen for the agent joining after us newRoom.on(RoomEvent.ParticipantConnected, (participant) => { if (newRoom === room && isAgent(participant)) markAgentFound(newRoom, participant); }); // Attach and play incoming audio tracks (agent TTS) newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => { if (newRoom === room && isAgent(participant) && track.kind === LivekitClient.Track.Kind.Audio) { const el = track.attach(); el.id = `audio-${participant.identity}-${publication.trackSid}`; document.body.appendChild(el); // Remote audio analyser drives the orb + visualizer while speaking setupRemoteAudioAnalyser(el); } }); newRoom.on(RoomEvent.TrackUnsubscribed, (track) => { track.detach().forEach((el) => el.remove()); }); // Browsers block audio playback until a user gesture; show an unlock // button if the page loads without one. newRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => { if (newRoom !== room) return; if (!newRoom.canPlaybackAudio) { const existing = document.getElementById("audioUnlockBtn"); if (!existing) { const btn = document.createElement("button"); btn.id = "audioUnlockBtn"; btn.className = "btn btn-primary"; btn.textContent = "Tap to enable audio"; btn.classList.add("audio-unlock"); btn.onclick = () => { newRoom.startAudio(); btn.remove(); }; document.body.appendChild(btn); } } else { const existing = document.getElementById("audioUnlockBtn"); if (existing) existing.remove(); } }); // Live text stream (word-by-word transcripts) registerTranscriptionStream(newRoom); // Data channel: transcripts (secondary source) + voice control newRoom.on(RoomEvent.DataReceived, (payload, participant, kind, topic) => { if (newRoom === room) handleDataPacket(payload, participant, topic); }); // Guard with `=== room` so a stale (retry-discarded) connection cannot // clobber UI state of the active one. newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => { if (participant === newRoom.localParticipant && newRoom === room) { if (![...progress.values()].some(item => item.status === "working")) statusEl.textContent = `Secure room · ${quality} connection`; } }); newRoom.on(RoomEvent.Disconnected, () => { if (newRoom !== room) return; stopConversation(); }); await newRoom.connect(LIVEKIT_URL, token, { autoSubscribe: true }); checkAttempt(attempt); const existing = findAgent(newRoom); if (existing) markAgentFound(newRoom, existing); return newRoom; } let connectionEpoch = 0; let tokenAbort = null; function checkAttempt(attempt) { if (attempt !== connectionEpoch || !HopeAuth.user || HopeAuth.user.status !== "ready") throw new Error("Connection cancelled"); } function waitFor(condition, timeout, attempt) { return new Promise((resolve, reject) => { const deadline = Date.now() + timeout; const tick = () => { try { checkAttempt(attempt); } catch (error) { reject(error); return; } if (condition()) resolve(true); else if (Date.now() >= deadline) resolve(false); else setTimeout(tick, 100); }; tick(); }); } function cleanupRoom() { const stale = room; room = null; agentParticipant = null; agentReady = false; if (stale) stale.disconnect().catch(() => {}); teardownAudio(); stopVisualLoop(); releaseWakeLock(); document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove()); document.getElementById("audioUnlockBtn")?.remove(); muteBtn.disabled = true; micMuted = false; updateMuteIcon(); } function stopConversation() { connectionEpoch++; tokenAbort?.abort(); tokenAbort = null; connectionPromise = null; cleanupRoom(); statusEl.textContent = "Conversation ended"; setState("idle"); startBtn.disabled = !HopeAuth.user || HopeAuth.user.status !== "ready"; stopBtn.disabled = true; for (const item of progress.values()) if (item.status === "working") { item.status = "failed"; item.message = "Conversation ended before completion."; } tasksBadge.hidden = true; } async function connectConversation() { if (!HopeAuth.user || HopeAuth.user.status !== "ready") throw new Error("Sign in with a ready account to connect."); if (connectionPromise) return connectionPromise; if (room && agentReady) return room; const attempt = ++connectionEpoch; startBtn.disabled = true; stopBtn.disabled = false; const promise = (async () => { try { setState("connecting"); statusEl.textContent = "Opening your secure room…"; checkAttempt(attempt); const tokenData = await fetchToken(attempt); checkAttempt(attempt); await createRoom(tokenData.token, attempt); checkAttempt(attempt); statusEl.textContent = "Inviting Hope into the room…"; if (!(await waitFor(() => !!findAgent(room), 15000, attempt))) throw new Error("Assistant did not join — try again"); if (!agentParticipant) markAgentFound(room, findAgent(room)); if (!(await waitFor(() => agentReady, 10000, attempt))) throw new Error("Assistant was not ready — try again"); checkAttempt(attempt); sendModeMessage(conversationMode); const active = room; if (conversationMode === "voice") { await active.startAudio(); checkAttempt(attempt); statusEl.textContent = "Waiting for microphone permission…"; await active.localParticipant.setMicrophoneEnabled(true); if (attempt !== connectionEpoch) { await active.localParticipant.setMicrophoneEnabled(false); throw new Error("Connection cancelled"); } checkAttempt(attempt); micMuted = false; setupMicAnalyser(); startVisualLoop(); statusEl.textContent = "Secure room · Microphone on"; } else { await active.localParticipant.setMicrophoneEnabled(false); checkAttempt(attempt); micMuted = true; stopVisualLoop(); statusEl.textContent = "Secure room · Text mode"; } muteBtn.disabled = conversationMode !== "voice"; updateMuteIcon(); setState("listening"); return room; } catch (error) { if (attempt === connectionEpoch) { cleanupRoom(); statusEl.textContent = `Error: ${error.message}`; setState("idle"); startBtn.disabled = !HopeAuth.user || HopeAuth.user.status !== "ready"; stopBtn.disabled = true; } throw error; } finally { if (attempt === connectionEpoch) connectionPromise = null; } })(); connectionPromise = promise; return promise; } startBtn.addEventListener("click", () => { connectConversation().catch(() => {}); }); // ── Text chat ────────────────────────────────────────────────────────────── function resizeChatInput() { chatInput.style.height = "auto"; chatInput.style.height = `${Math.min(chatInput.scrollHeight, 144)}px`; } chatInput.addEventListener("input", () => { resizeChatInput(); sendChatBtn.disabled = !chatInput.value.trim(); }); chatInput.addEventListener("keydown", (event) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); chatComposer.requestSubmit(); } }); chatComposer.addEventListener("submit", async (event) => { event.preventDefault(); const text = chatInput.value.trim(); if (!text || !HopeAuth.user || HopeAuth.user.status !== "ready") return; sendChatBtn.disabled = true; chatInput.disabled = true; try { if (conversationMode !== "text") await setConversationMode("text"); await connectConversation(); setState("thinking"); await room.localParticipant.sendText(text, { topic: "lk.chat" }); chatInput.value = ""; resizeChatInput(); } catch (err) { console.error("Text message failed:", err); statusEl.textContent = `Message not sent: ${err.message}`; setState(room ? "listening" : "idle"); } finally { chatInput.disabled = !HopeAuth.user || HopeAuth.user.status !== "ready"; sendChatBtn.disabled = !HopeAuth.user || !chatInput.value.trim(); chatInput.focus(); } }); // ── Stop ──────────────────────────────────────────────────────────────────── stopBtn.addEventListener("click", stopConversation); // ── Mic mute toggle ───────────────────────────────────────────────────────── muteBtn.addEventListener("click", async () => { if (!room || !room.localParticipant) return; const nextMuted = !micMuted; try { await room.localParticipant.setMicrophoneEnabled(!nextMuted); micMuted = nextMuted; updateMuteIcon(); } catch (err) { console.warn("Microphone toggle failed:", err); statusEl.textContent = "Microphone unavailable in this browser"; } }); function updateMuteIcon() { const muted = muteBtn.classList.toggle("muted", micMuted); muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone"); muteBtn.title = muted ? "Unmute microphone" : "Mute microphone"; } // ── Helper: fetch a signed access token from the token endpoint ──────────── async function fetchToken(attempt) { checkAttempt(attempt); tokenAbort = new AbortController(); const data = await HopeAuth.request("/token", { method: "POST", body: JSON.stringify({}), signal: tokenAbort.signal }); checkAttempt(attempt); if (!data.token || typeof data.token !== "string") throw new Error("Token endpoint did not return a room token"); return data; } // ── Helper: send voice selection to the agent via data channel ───────────── function sendVoiceMessage(voice, targetRoom) { const r = targetRoom || room; if (!r || !r.localParticipant) return; const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice })); r.localParticipant.publishData(payload, { topic: "voice-control", reliable: true, }).catch(() => {}); } function sendModeMessage(mode, targetRoom) { const r = targetRoom || room; if (!r || !r.localParticipant) return; const payload = new TextEncoder().encode(JSON.stringify({ type: "set_mode", mode })); r.localParticipant.publishData(payload, { topic: "conversation-control", reliable: true, }).catch(() => {}); } HopeAuth.onChange((user, changed) => { if (changed || !user || user.status !== "ready") stopConversation(); if (changed || settingsAccount !== (user ? String(user.id) : null)) resetModelSettings(user); if (changed) { voiceCatalogGeneration++; voiceCatalogRequest = null; VOICES = [{ id: OMNI_DEFAULT, name: "Ava", locale: "en-US", gender: "Female", tag: "Dragon HD Omni" }]; defaultVoice = OMNI_DEFAULT; voiceSearchEl.value = ""; if (user) loadVoiceCatalog(); historyKey = user ? `hope_history:${location.origin}:${encodeURIComponent(user.id)}` : null; history = loadHistory(); renderHistory(); progress.clear(); tasksList.replaceChildren(); tasksBadge.hidden = true; chatInput.value = ""; sendChatBtn.disabled = true; setTasksOpen(false); closeSheet(); } startBtn.disabled = !user || user.status !== "ready" || !!room || !!connectionPromise; chatInput.disabled = !user || user.status !== "ready"; }); buildLedWall();