// Voice Assistant — LiveKit client const { Room, RoomEvent } = LivekitClient; // ── Config ────────────────────────────────────────────────────────────────── // LiveKit is reached through the same origin as the page (nginx proxies // /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed // content when the UI is served over HTTPS. const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`; const ROOM_NAME = "voice-room"; const AGENT_NAME = "voice-assistant"; const VOICES = [ { id: "en-US-Ava:DragonHDOmniLatestNeural", name: "Ava", tag: "Dragon HD Omni" }, { id: "en-US-JennyNeural", name: "Jenny" }, { id: "en-US-GuyNeural", name: "Guy" }, { id: "en-US-AndrewNeural", name: "Andrew" }, { id: "en-US-AriaNeural", name: "Aria" }, { id: "en-US-EmmaNeural", name: "Emma" }, { id: "en-US-EricNeural", name: "Eric" }, { id: "en-US-BrianNeural", name: "Brian" }, { id: "en-US-AshleyNeural", name: "Ashley" }, { id: "en-US-RichardNeural", name: "Richard" }, { id: "en-US-TinaNeural", name: "Tina" }, { id: "en-US-SteffanNeural", name: "Steffan" }, ]; const HISTORY_KEY = "voice_history"; const HISTORY_MAX = 50; const MODE_KEY = "hope_conversation_mode"; const MODEL_KEY = "preferred_model"; // ── State ─────────────────────────────────────────────────────────────────── let room = null; let agentParticipant = null; let agentReady = false; let uiState = "idle"; // idle | connecting | listening | thinking | speaking let micLevel = 0; // 0..1, smoothed let agentAudioLevel = 0; // 0..1, smoothed let userSpeaking = false; // local VAD-ish flag from mic level let wakeLock = null; let conversationMode = localStorage.getItem(MODE_KEY) === "text" ? "text" : "voice"; let connectionPromise = null; // ── Model (brain) state — populated from the agent's data channel ────────── let availableModels = []; // ids published by the agent let selectedModel = localStorage.getItem(MODEL_KEY) || null; function escapeHtml(s) { const d = document.createElement("div"); d.textContent = s || ""; return d.innerHTML; } // ── DOM ───────────────────────────────────────────────────────────────────── const startBtn = document.getElementById("startBtn"); const stopBtn = document.getElementById("stopBtn"); const muteBtn = document.getElementById("muteBtn"); let micMuted = false; const statusEl = document.getElementById("status"); const messagesEl = document.getElementById("messages"); const orbEl = document.getElementById("orb"); const stateLabelEl = document.getElementById("stateLabel"); const thinkingChipEl = document.getElementById("thinkingChip"); const bargeHintEl = document.getElementById("bargeHint"); const micMeterFillEl = document.getElementById("micMeterFill"); const settingsBtn = document.getElementById("settingsBtn"); const settingsSheet = document.getElementById("settingsSheet"); const sheetBackdrop = document.getElementById("sheetBackdrop"); const closeSheetBtn = document.getElementById("closeSheetBtn"); const voiceListEl = document.getElementById("voiceList"); const modelListEl = document.getElementById("modelList"); const modelNoteEl = document.getElementById("modelNote"); const clearBtn = document.getElementById("clearBtn"); const tasksBtn = document.getElementById("tasksBtn"); const tasksPanel = document.getElementById("tasksPanel"); const tasksList = document.getElementById("tasksList"); const tasksBadge = document.getElementById("tasksBadge"); const closeTasksBtn = document.getElementById("closeTasksBtn"); const voiceModeBtn = document.getElementById("voiceModeBtn"); const textModeBtn = document.getElementById("textModeBtn"); const modeDescriptionEl = document.getElementById("modeDescription"); const transcriptModeLabelEl = document.getElementById("transcriptModeLabel"); const chatComposer = document.getElementById("chatComposer"); const chatInput = document.getElementById("chatInput"); const sendChatBtn = document.getElementById("sendChatBtn"); // ── Background tasks state ────────────────────────────────────────────────── const tasks = new Map(); // task_id -> {id, description, status, steps: [], result, error} function updateTasksBadge() { const active = [...tasks.values()].filter(t => t.status === "running" || t.status === "pending").length; if (active > 0) { tasksBadge.hidden = false; tasksBadge.textContent = String(active); } else { tasksBadge.hidden = true; } } function renderTasks() { const items = [...tasks.values()].sort((a, b) => (b.created_at || 0) - (a.created_at || 0)); if (items.length === 0) { tasksList.innerHTML = '
Nothing runningAsk Hope to research or compare something.
'; return; } tasksList.innerHTML = ""; for (const t of items) { const card = document.createElement("div"); card.className = "task-card"; const statusClass = t.status || "pending"; card.innerHTML = `
${escapeHtml(t.description)}
${t.result ? `
${escapeHtml(t.result)}
` : ""} ${t.error ? `
${escapeHtml(t.error)}
` : ""} `; tasksList.appendChild(card); } } function handleTaskEvent(data) { const { task_id, event, role, content, result, error, task } = data; if (event === "created" && task) { tasks.set(task_id, { ...task, steps: [] }); } else if (event === "step") { const t = tasks.get(task_id); if (t) { t.steps.push({ role, content }); t.status = "running"; } } else if (event === "completed") { const t = tasks.get(task_id); if (t) { t.status = "completed"; t.result = result; } } else if (event === "failed") { const t = tasks.get(task_id); if (t) { t.status = "failed"; t.error = error; } } updateTasksBadge(); renderTasks(); } function setTasksOpen(open) { tasksPanel.classList.toggle("open", open); tasksBtn.setAttribute("aria-expanded", String(open)); } tasksBtn.setAttribute("aria-expanded", "false"); tasksBtn.addEventListener("click", () => setTasksOpen(!tasksPanel.classList.contains("open"))); closeTasksBtn.addEventListener("click", () => setTasksOpen(false)); // ── Voice / text mode ────────────────────────────────────────────────────── function updateModeUI() { const isText = conversationMode === "text"; document.body.dataset.mode = conversationMode; voiceModeBtn.classList.toggle("active", !isText); textModeBtn.classList.toggle("active", isText); voiceModeBtn.setAttribute("aria-pressed", String(!isText)); textModeBtn.setAttribute("aria-pressed", String(isText)); modeDescriptionEl.textContent = isText ? "Type when quiet is better. Switch back to voice without losing the thread." : "Speak naturally. I’ll listen, remember, and answer in real time."; transcriptModeLabelEl.textContent = isText ? "Shared voice and text history" : "Live voice transcript"; startBtn.innerHTML = isText ? 'Connect to Hope' : 'Start conversation'; if (isText) { setState(room ? "listening" : "idle"); window.setTimeout(() => chatInput.focus(), 0); } else { // Refresh mode-specific copy even when the underlying state is unchanged. setState(uiState); } } async function setConversationMode(mode) { if (mode !== "voice" && mode !== "text") return; const previousMode = conversationMode; conversationMode = mode; localStorage.setItem(MODE_KEY, mode); updateModeUI(); if (mode === "voice" && previousMode === "text") { chatInput.blur(); window.setTimeout(() => { document.querySelector(".workspace").scrollIntoView({ block: "start", behavior: "smooth" }); }, 0); } if (!room || !room.localParticipant) return; sendModeMessage(mode); if (mode === "text") { await room.localParticipant.setMicrophoneEnabled(false); micMuted = true; muteBtn.disabled = true; updateMuteIcon(); stopVisualLoop(); statusEl.textContent = "Secure room · Text mode"; } else { micMuted = false; muteBtn.disabled = false; updateMuteIcon(); startVisualLoop(); try { await room.startAudio(); await room.localParticipant.setMicrophoneEnabled(true); setupMicAnalyser(); statusEl.textContent = "Secure room · Microphone on"; } catch (err) { // Keep the conversation usable even if this browser denies or does // not implement microphone capture. The mute control can retry. console.warn("Microphone could not be enabled:", err); micMuted = true; updateMuteIcon(); statusEl.textContent = "Voice mode · Microphone unavailable"; } } } voiceModeBtn.addEventListener("click", () => setConversationMode("voice").catch(console.error)); textModeBtn.addEventListener("click", () => setConversationMode("text").catch(console.error)); // ── Audio analysers (mic + remote agent audio) ───────────────────────────── let micCtx = null; let micAnalyser = null; let micBuf = null; let remoteCtx = null; let remoteAnalyser = null; let remoteFreqBuf = null; function setupMicAnalyser() { try { const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone); const mst = pub && pub.track && pub.track.mediaStreamTrack; if (mst) { micCtx = new AudioContext(); micAnalyser = micCtx.createAnalyser(); micAnalyser.fftSize = 512; micBuf = new Float32Array(micAnalyser.fftSize); micCtx.createMediaStreamSource(new MediaStream([mst])).connect(micAnalyser); } } catch (e) { /* fall back to SDK audioLevel in the meter loop */ } } function setupRemoteAudioAnalyser(audioEl) { try { if (!audioEl.srcObject) return; remoteCtx = new AudioContext(); remoteAnalyser = remoteCtx.createAnalyser(); remoteAnalyser.fftSize = 256; remoteFreqBuf = new Uint8Array(remoteAnalyser.frequencyBinCount); remoteCtx.createMediaStreamSource(audioEl.srcObject).connect(remoteAnalyser); } catch (e) { /* analyser optional */ } } function teardownAudio() { for (const ctx of [micCtx, remoteCtx]) { if (ctx && ctx.state !== "closed") ctx.close().catch(() => {}); } micCtx = micAnalyser = micBuf = null; remoteCtx = remoteAnalyser = remoteFreqBuf = null; } function readMicLevel() { let level = 0; if (micAnalyser && micBuf) { micAnalyser.getFloatTimeDomainData(micBuf); for (let i = 0; i < micBuf.length; i++) level = Math.max(level, Math.abs(micBuf[i])); } else if (room && room.localParticipant) { level = room.localParticipant.audioLevel || 0; } // Smooth so the orb pulses rather than flickers micLevel = micLevel * 0.6 + level * 0.4; return micLevel; } function readAgentAudioLevel() { let level = 0; if (remoteAnalyser && remoteFreqBuf) { remoteAnalyser.getByteFrequencyData(remoteFreqBuf); for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]); level /= 255; } else if (agentParticipant && agentParticipant.audioLevel !== undefined) { level = agentParticipant.audioLevel || 0; } agentAudioLevel = agentAudioLevel * 0.6 + level * 0.4; return agentAudioLevel; } // ── State machine ─────────────────────────────────────────────────────────── const STATE_LABELS = { voice: { idle: "Ready when you are", connecting: "Opening a secure room", listening: "Listening", thinking: "Thinking it through", speaking: "Hope is speaking", }, text: { idle: "Ready to type", connecting: "Opening a secure room", listening: "Text chat ready", thinking: "Hope is writing", speaking: "Hope replied", }, }; function setState(next) { uiState = next; orbEl.dataset.state = next; stateLabelEl.textContent = STATE_LABELS[conversationMode][next] || next; // Thinking timer: show the "thinking..." chip if it lingers past 2s if (next === "thinking") { thinkingChipEl.hidden = true; setTimeout(() => { if (uiState === "thinking") thinkingChipEl.hidden = false; }, 2000); } else { thinkingChipEl.hidden = true; } // Barge-in hint only while the agent is speaking bargeHintEl.hidden = next !== "speaking"; if (next === "idle") { micLevel = 0; agentAudioLevel = 0; userSpeaking = false; } } updateModeUI(); function inferState() { if (!room || !room.localParticipant) return "idle"; if (!agentParticipant) return "connecting"; readMicLevel(); readAgentAudioLevel(); userSpeaking = micLevel > 0.035; if (agentAudioLevel > 0.02) return "speaking"; if (userSpeaking) return "listening"; // Once agent audio ends, return to listening. The previous implementation // converted speaking → thinking forever, leaving the UI apparently frozen. if (uiState === "speaking") return "listening"; if (uiState === "thinking") return "thinking"; return "listening"; } // ── Visual loop: drives orb + meter + state inference ─────────────────────── let visualTimer = null; function startVisualLoop() { if (visualTimer) clearInterval(visualTimer); visualTimer = setInterval(() => { if (!room || !room.localParticipant) return; setState(inferState()); // Mic meter (thin bar under the status line) micMeterFillEl.style.width = `${Math.min(100, Math.round(micLevel * 250))}%`; // Orb pulse: scale/opacity from whichever stream is active if (uiState === "speaking") { orbEl.style.setProperty("--pulse-scale", (1 + agentAudioLevel * 0.35).toFixed(3)); orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.55 + agentAudioLevel).toFixed(3)); } else if (uiState === "listening" && userSpeaking) { orbEl.style.setProperty("--pulse-scale", (1 + micLevel * 0.4).toFixed(3)); orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.5 + micLevel).toFixed(3)); } else { orbEl.style.setProperty("--pulse-scale", "1"); orbEl.style.setProperty("--pulse-opacity", ""); } // Agent audio visualizer bars on the latest agent bubble const viz = messagesEl.querySelector(".message-row.agent:last-of-type .audio-viz"); if (viz) { let level = 0; if (remoteAnalyser && remoteFreqBuf) { for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]); level /= 255; } const bars = viz.children; for (let i = 0; i < bars.length; i++) { // Per-bar phase offset so the bars dance rather than move in lockstep const h = Math.max(0.15, level * (0.6 + 0.4 * Math.sin(Date.now() / 120 + i * 1.3))); bars[i].style.height = `${h * 100}%`; } } }, 80); } function stopVisualLoop() { if (visualTimer) clearInterval(visualTimer); visualTimer = null; micMeterFillEl.style.width = "0%"; orbEl.style.setProperty("--pulse-scale", "1"); orbEl.style.setProperty("--pulse-opacity", ""); } // ── Conversation history (localStorage) ───────────────────────────────────── function loadHistory() { try { const raw = localStorage.getItem(HISTORY_KEY); if (!raw) return []; const arr = JSON.parse(raw); return Array.isArray(arr) ? arr.slice(-HISTORY_MAX) : []; } catch (e) { return []; } } function persistHistory() { try { localStorage.setItem(HISTORY_KEY, JSON.stringify(history.slice(-HISTORY_MAX))); } catch (e) { /* storage full or unavailable */ } } let history = loadHistory(); const DELIVERY_MARKERS = /\[(?:appreciative|calm|concerned|curious|encouraging|excited|happy|joking|quiet|reassuring|reflective|sad|serious|surprised|sympathetic|urgent|whispering|laughter|sighing|breathing|neutral)\]\s*/gi; function cleanSpokenText(text) { return String(text || "").replace(DELIVERY_MARKERS, "").trim(); } function formatTimestamp(ts) { if (!ts) return ""; return new Date(ts).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }); } function renderMessage(msg) { const wrap = document.createElement("div"); wrap.className = `message-row ${msg.role}`; wrap.dataset.ts = formatTimestamp(msg.ts); if (msg.role === "agent") { const dot = document.createElement("span"); dot.className = "avatar-dot"; wrap.appendChild(dot); } const bubble = document.createElement("div"); bubble.className = "bubble"; bubble.textContent = cleanSpokenText(msg.text); if (msg.role === "agent") { const viz = document.createElement("span"); viz.className = "audio-viz"; for (let i = 0; i < 5; i++) viz.appendChild(document.createElement("i")); bubble.appendChild(viz); } wrap.appendChild(bubble); messagesEl.appendChild(wrap); messagesEl.scrollTop = messagesEl.scrollHeight; } function addMessage(role, text) { clearPartials(); const msg = { role, text, ts: Date.now() }; history.push(msg); persistHistory(); renderMessage(msg); return msg; } // Partial transcripts render greyed and update in place until a final arrives. function addPartial(role, text) { const rows = messagesEl.querySelectorAll(".message-row.partial"); let last = null; for (const r of rows) last = r; if (last && last.dataset.role === role) { last.querySelector(".bubble").textContent = cleanSpokenText(text); messagesEl.scrollTop = messagesEl.scrollHeight; return; } const wrap = document.createElement("div"); wrap.className = `message-row ${role} partial`; wrap.dataset.role = role; if (role === "agent") { const dot = document.createElement("span"); dot.className = "avatar-dot"; wrap.appendChild(dot); } const bubble = document.createElement("div"); bubble.className = "bubble partial"; bubble.textContent = cleanSpokenText(text); wrap.appendChild(bubble); messagesEl.appendChild(wrap); messagesEl.scrollTop = messagesEl.scrollHeight; } function clearPartials() { messagesEl.querySelectorAll(".message-row.partial").forEach((el) => el.remove()); } function renderHistory() { messagesEl.innerHTML = ""; for (const msg of history) renderMessage(msg); messagesEl.scrollTop = messagesEl.scrollHeight; } clearBtn.addEventListener("click", () => { history = []; persistHistory(); clearPartials(); renderHistory(); }); renderHistory(); // ── Transcripts ───────────────────────────────────────────────────────────── // Primary: livekit-agents built-in text stream (word-by-word when available). function registerTranscriptionStream(rm) { try { rm.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => { const text = await reader.readAll(); if (!text || !text.trim()) return; const attrs = (reader.info && reader.info.attributes) || {}; const trackId = attrs["lk.transcribed_track_id"] || ""; // Distinguish user vs agent by the transcribed track's owner. let role = "agent"; if (participantInfo && participantInfo.identity === AGENT_NAME) { role = "agent"; } else if (trackId) { for (const p of rm.remoteParticipants.values()) { const pub = p.getTrackPublication(trackId); if (pub) { role = isAgent(p) ? "agent" : "user"; break; } } } // Use partial rendering: updates the same row in place until final. addPartial(role, text.trim()); }); } catch (e) { // Older SDKs may not support registerTextStreamHandler — the data // channel handler below still covers transcripts. console.warn("text stream registration failed:", e); } } // Secondary: agent publishes {type:"transcript"} on topic "transcript". function handleDataPacket(payload, participant, topic) { try { const msg = JSON.parse(new TextDecoder().decode(payload)); if (msg.type === "agent_ready") { agentReady = true; } else if (msg.type === "transcript") { addMessage(msg.role || "agent", msg.text); if ((msg.role || "agent") === "agent") { if (conversationMode === "text") { setState("listening"); chatInput.focus(); } else if (uiState === "thinking") { // Text arrives even when headless playback or an analyser is // unavailable; use it to release the thinking state. setState("speaking"); } } } else if (msg.type === "task_event") { handleTaskEvent(msg); } else if (msg.type === "models") { availableModels = Array.isArray(msg.models) ? msg.models : []; // Don't clobber a saved preference — the agent acks the applied // model separately via model_changed. if (!selectedModel && msg.selected) selectedModel = msg.selected; renderModelList(); } else if (msg.type === "model_changed") { if (msg.model) { selectedModel = msg.model; localStorage.setItem(MODEL_KEY, selectedModel); renderModelList(); } } // set_voice/set_model messages flow the other way; nothing else to do client-side. } catch (e) { console.warn("bad data packet", e); } } // ── Voice settings sheet ──────────────────────────────────────────────────── function currentVoiceId() { const saved = localStorage.getItem("preferred_voice"); return VOICES.some((v) => v.id === saved) ? saved : VOICES[0].id; } function renderVoiceList() { voiceListEl.innerHTML = ""; const active = currentVoiceId(); for (const v of VOICES) { const card = document.createElement("div"); card.className = "voice-card" + (v.id === active ? " active" : ""); card.dataset.voice = v.id; const name = document.createElement("span"); name.className = "voice-name"; name.textContent = v.name + (v.tag ? ` (${v.tag})` : ""); const selected = v.id === active; card.setAttribute("role", "radio"); card.setAttribute("aria-checked", String(selected)); card.tabIndex = 0; card.appendChild(name); card.addEventListener("click", () => selectVoice(v.id)); card.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); selectVoice(v.id); } }); voiceListEl.appendChild(card); } } function selectVoice(voiceId) { localStorage.setItem("preferred_voice", voiceId); renderVoiceList(); if (room && agentParticipant) { sendVoiceMessage(voiceId); const v = VOICES.find((x) => x.id === voiceId); addMessage("system", `Voice changed to ${v ? v.name : voiceId}`); } } // ── Model (brain) settings ────────────────────────────────────────────────── function currentModelId() { if (availableModels.length === 0) return selectedModel; if (selectedModel && availableModels.includes(selectedModel)) return selectedModel; return availableModels[0]; } function renderModelList() { modelListEl.innerHTML = ""; if (!availableModels.length) { modelNoteEl.hidden = false; modelNoteEl.textContent = room ? "Loading available models…" : "Start a conversation to load the model list."; return; } modelNoteEl.hidden = true; const active = currentModelId(); for (const id of availableModels) { const card = document.createElement("div"); card.className = "voice-card" + (id === active ? " active" : ""); card.dataset.model = id; card.setAttribute("role", "radio"); card.setAttribute("aria-checked", String(id === active)); card.tabIndex = 0; const name = document.createElement("span"); name.className = "voice-name"; name.textContent = id; card.appendChild(name); card.addEventListener("click", () => selectModel(id)); card.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); selectModel(id); } }); modelListEl.appendChild(card); } } function selectModel(modelId) { selectedModel = modelId; localStorage.setItem(MODEL_KEY, modelId); renderModelList(); if (room && agentParticipant) { sendModelMessage(modelId); addMessage("system", `Brain changed to ${modelId}`); } } function openSheet() { renderVoiceList(); renderModelList(); sheetBackdrop.hidden = false; settingsSheet.classList.add("open"); settingsBtn.setAttribute("aria-expanded", "true"); settingsSheet.querySelector(".voice-card")?.focus(); } function closeSheet() { settingsSheet.classList.remove("open"); settingsBtn.setAttribute("aria-expanded", "false"); window.setTimeout(() => { sheetBackdrop.hidden = true; }, 240); settingsBtn.focus(); } settingsBtn.setAttribute("aria-expanded", "false"); settingsBtn.addEventListener("click", openSheet); closeSheetBtn.addEventListener("click", closeSheet); sheetBackdrop.addEventListener("click", closeSheet); document.addEventListener("keydown", (event) => { if (event.key !== "Escape") return; if (settingsSheet.classList.contains("open")) closeSheet(); if (tasksPanel.classList.contains("open")) setTasksOpen(false); if (mediaPanel.classList.contains("open")) setMediaOpen(false); }); // ── Wake lock ─────────────────────────────────────────────────────────────── async function acquireWakeLock() { try { if ("wakeLock" in navigator) { wakeLock = await navigator.wakeLock.request("screen"); } } catch (e) { /* not critical */ } } function releaseWakeLock() { if (wakeLock) { wakeLock.release().catch(() => {}); wakeLock = null; } } // ── Agent detection ───────────────────────────────────────────────────────── // The agent may already be in the room before we join (it persists between // page loads), so scan remoteParticipants after connect AND watch for new // joins — ParticipantConnected does not fire for pre-existing participants. function isAgent(participant) { // kind is the numeric proto enum; ParticipantKind.AGENT === 4. // Prefer the SDK's own getter where available. if (participant.isAgent !== undefined && participant.isAgent !== null) { return participant.isAgent; } return participant.identity === AGENT_NAME || participant.kind === LivekitClient.ParticipantKind.AGENT; } function findAgent(rm) { if (!rm) return null; for (const p of rm.remoteParticipants.values()) { if (isAgent(p)) return p; } return null; } function markAgentFound(rm, participant) { agentParticipant = participant; statusEl.textContent = conversationMode === "text" ? "Hope joined · Text chat ready" : "Hope joined · Preparing microphone…"; // Synchronize voice, brain and interaction mode when the agent joins. sendVoiceMessage(currentVoiceId(), rm); if (selectedModel) sendModelMessage(selectedModel, rm); sendModeMessage(conversationMode, rm); } function createRoom(token) { const newRoom = new Room({ adaptiveStream: true, dynacast: true, }); // Listen for the agent joining after us newRoom.on(RoomEvent.ParticipantConnected, (participant) => { if (isAgent(participant)) markAgentFound(newRoom, participant); }); // Attach and play incoming audio tracks (agent TTS) newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => { if (track.kind === LivekitClient.Track.Kind.Audio) { const el = track.attach(); el.id = `audio-${participant.identity}`; document.body.appendChild(el); // Remote audio analyser drives the orb + visualizer while speaking setupRemoteAudioAnalyser(el); } }); newRoom.on(RoomEvent.TrackUnsubscribed, (track) => { track.detach().forEach((el) => el.remove()); }); // Browsers block audio playback until a user gesture; show an unlock // button if the page loads without one. newRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => { if (!newRoom.canPlaybackAudio) { const existing = document.getElementById("audioUnlockBtn"); if (!existing) { const btn = document.createElement("button"); btn.id = "audioUnlockBtn"; btn.className = "btn btn-primary"; btn.textContent = "Tap to enable audio"; btn.style.cssText = "position:fixed;bottom:2rem;left:50%;transform:translateX(-50%);z-index:999;padding:1rem 2rem;font-size:1.1rem;border-radius:999px;box-shadow:0 4px 24px rgba(0,0,0,.5)"; btn.onclick = () => { newRoom.startAudio(); btn.remove(); }; document.body.appendChild(btn); } } else { const existing = document.getElementById("audioUnlockBtn"); if (existing) existing.remove(); } }); // Live text stream (word-by-word transcripts) registerTranscriptionStream(newRoom); // Data channel: transcripts (secondary source) + voice control newRoom.on(RoomEvent.DataReceived, (payload, participant, kind, topic) => { handleDataPacket(payload, participant, topic); }); // Guard with `=== room` so a stale (retry-discarded) connection cannot // clobber UI state of the active one. newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => { if (participant === newRoom.localParticipant && newRoom === room) { statusEl.textContent = `Secure room · ${quality} connection`; } }); newRoom.on(RoomEvent.Disconnected, () => { if (newRoom !== room) return; teardownAudio(); stopVisualLoop(); releaseWakeLock(); statusEl.textContent = "Conversation ended"; setState("idle"); startBtn.disabled = false; stopBtn.disabled = true; agentParticipant = null; agentReady = false; // Leaving the conversation hands the room back to the music player. resumePlaybackAfterTalk(); }); return newRoom.connect(LIVEKIT_URL, token, { autoSubscribe: true, }).then(() => { // Pick up an agent that was already in the room before we joined const existing = findAgent(newRoom); if (existing) markAgentFound(newRoom, existing); return newRoom; }); } function waitForAgentReady(timeoutMs) { return new Promise((resolve) => { if (agentReady) return resolve(true); const deadline = Date.now() + timeoutMs; const timer = setInterval(() => { if (agentReady) { clearInterval(timer); resolve(true); } else if (Date.now() > deadline) { clearInterval(timer); resolve(false); } }, 100); }); } function waitForAgent(timeoutMs) { return new Promise((resolve) => { if (findAgent(room)) return resolve(true); const deadline = Date.now() + timeoutMs; const timer = setInterval(() => { const found = findAgent(room); if (found) { if (!agentParticipant) markAgentFound(room, found); clearInterval(timer); resolve(true); } else if (Date.now() > deadline) { clearInterval(timer); resolve(false); } }, 250); }); } async function connectConversation() { if (room && agentParticipant) return room; if (connectionPromise) return connectionPromise; connectionPromise = (async () => { try { setState("connecting"); statusEl.textContent = "Opening your secure room…"; await acquireWakeLock(); // Get a signed access token from the token endpoint. const tokenData = await fetchToken(ROOM_NAME); await enterMediaScope(tokenData.room); room = await createRoom(tokenData.token); // The agent is dispatched when we join. If it misses the dispatch // (for example during a server restart), reconnect once. statusEl.textContent = "Inviting Hope into the room…"; if (!(await waitForAgent(10000))) { statusEl.textContent = "Hope is taking a moment · Retrying…"; const stale = room; room = null; agentParticipant = null; agentReady = false; await stale.disconnect(); const freshTokenData = await fetchToken(ROOM_NAME); await enterMediaScope(freshTokenData.room); room = await createRoom(freshTokenData.token); if (!(await waitForAgent(15000))) { throw new Error("Assistant did not join — reload the page and try again"); } } statusEl.textContent = "Hope joined · Finishing setup…"; if (!(await waitForAgentReady(10000))) { throw new Error("Assistant joined but was not ready — try again"); } sendModeMessage(conversationMode); if (conversationMode === "voice") { await room.startAudio(); statusEl.textContent = "Waiting for microphone permission…"; await room.localParticipant.setMicrophoneEnabled(true); micMuted = false; setupMicAnalyser(); startVisualLoop(); statusEl.textContent = "Secure room · Microphone on"; } else { await room.localParticipant.setMicrophoneEnabled(false); micMuted = true; stopVisualLoop(); statusEl.textContent = "Secure room · Text mode"; } startBtn.disabled = true; stopBtn.disabled = false; muteBtn.disabled = conversationMode !== "voice"; updateMuteIcon(); setState("listening"); return room; } catch (err) { console.error("Connection failed:", err); statusEl.textContent = `Error: ${err.message}`; setState("idle"); releaseWakeLock(); throw err; } finally { connectionPromise = null; } })(); return connectionPromise; } startBtn.addEventListener("click", () => { connectConversation().catch(() => {}); }); // ── Text chat ────────────────────────────────────────────────────────────── function resizeChatInput() { chatInput.style.height = "auto"; chatInput.style.height = `${Math.min(chatInput.scrollHeight, 144)}px`; } chatInput.addEventListener("input", () => { resizeChatInput(); sendChatBtn.disabled = !chatInput.value.trim(); }); chatInput.addEventListener("keydown", (event) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); chatComposer.requestSubmit(); } }); chatComposer.addEventListener("submit", async (event) => { event.preventDefault(); const text = chatInput.value.trim(); if (!text) return; sendChatBtn.disabled = true; chatInput.disabled = true; try { if (conversationMode !== "text") await setConversationMode("text"); await connectConversation(); setState("thinking"); await room.localParticipant.sendText(text, { topic: "lk.chat" }); chatInput.value = ""; resizeChatInput(); } catch (err) { console.error("Text message failed:", err); statusEl.textContent = `Message not sent: ${err.message}`; setState("listening"); } finally { chatInput.disabled = false; sendChatBtn.disabled = !chatInput.value.trim(); chatInput.focus(); } }); // ── Stop ──────────────────────────────────────────────────────────────────── stopBtn.addEventListener("click", () => { if (room) { room.disconnect(); room = null; agentParticipant = null; agentReady = false; } teardownAudio(); stopVisualLoop(); releaseWakeLock(); document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove()); statusEl.textContent = "Conversation ended"; setState("idle"); startBtn.disabled = false; stopBtn.disabled = true; muteBtn.disabled = true; micMuted = false; updateMuteIcon(); // Ending the conversation resumes music that was paused for the talk. resumePlaybackAfterTalk(); }); // ── Mic mute toggle ───────────────────────────────────────────────────────── muteBtn.addEventListener("click", async () => { if (!room || !room.localParticipant) return; const nextMuted = !micMuted; try { await room.localParticipant.setMicrophoneEnabled(!nextMuted); micMuted = nextMuted; updateMuteIcon(); } catch (err) { console.warn("Microphone toggle failed:", err); statusEl.textContent = "Microphone unavailable in this browser"; } }); function updateMuteIcon() { const muted = muteBtn.classList.toggle("muted", micMuted); muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone"); muteBtn.title = muted ? "Unmute microphone" : "Mute microphone"; } // ── Helper: fetch a signed access token from the token endpoint ──────────── async function fetchToken(roomName) { const resp = await fetch("/token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ room: roomName }), }); if (!resp.ok) throw new Error(`Token endpoint returned ${resp.status}`); const data = await resp.json(); return { token: data.token, room: data.room || null }; } // ── Helper: send voice selection to the agent via data channel ───────────── function sendVoiceMessage(voice, targetRoom) { const r = targetRoom || room; if (!r || !r.localParticipant) return; const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice })); r.localParticipant.publishData(payload, { topic: "voice-control", reliable: true, }); } function sendModeMessage(mode, targetRoom) { const r = targetRoom || room; if (!r || !r.localParticipant) return; const payload = new TextEncoder().encode(JSON.stringify({ type: "set_mode", mode })); r.localParticipant.publishData(payload, { topic: "conversation-control", reliable: true, }); } // ── Helper: send model selection to the agent via data channel ───────────── function sendModelMessage(model, targetRoom) { const r = targetRoom || room; if (!r || !r.localParticipant || !model) return; const payload = new TextEncoder().encode(JSON.stringify({ type: "set_model", model })); r.localParticipant.publishData(payload, { topic: "voice-control", reliable: true, }); } // ── Media player ──────────────────────────────────────────────────────────── // The browser