snowspeeder
68f4c461c8
Rebuild & Deploy / rebuild (push) Successful in 37s
- Swap the CSS orb for a 21x21 LED dot matrix on a glass panel, matching the livekit.com LED-array hero visual. - Pulse driven by the agent's rendered audio: an expanding radial bloom brightens and widens with remote-audio level, dimmer when listening on the mic, faint static matrix when idle, and a subtle warm breathing state while thinking. - State-tinted palettes: cool azure when speaking, blue when listening, amber when thinking, dim slate when idle. - Drop the unused orb CSS-variable pulse plumbing and the presence-breathe / orbit-turn keyframes; the wall renders per-frame from the analyser.
1666 lines
64 KiB
JavaScript
1666 lines
64 KiB
JavaScript
// Voice Assistant — LiveKit client
|
||
const { Room, RoomEvent } = LivekitClient;
|
||
|
||
// ── Config ──────────────────────────────────────────────────────────────────
|
||
// LiveKit is reached through the same origin as the page (nginx proxies
|
||
// /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed
|
||
// content when the UI is served over HTTPS.
|
||
const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`;
|
||
const ROOM_NAME = "voice-room";
|
||
const AGENT_NAME = "voice-assistant";
|
||
|
||
const VOICES = [
|
||
{ id: "en-US-Ava:DragonHDOmniLatestNeural", name: "Ava", tag: "Dragon HD Omni" },
|
||
{ id: "en-US-JennyNeural", name: "Jenny" },
|
||
{ id: "en-US-GuyNeural", name: "Guy" },
|
||
{ id: "en-US-AndrewNeural", name: "Andrew" },
|
||
{ id: "en-US-AriaNeural", name: "Aria" },
|
||
{ id: "en-US-EmmaNeural", name: "Emma" },
|
||
{ id: "en-US-EricNeural", name: "Eric" },
|
||
{ id: "en-US-BrianNeural", name: "Brian" },
|
||
{ id: "en-US-AshleyNeural", name: "Ashley" },
|
||
{ id: "en-US-RichardNeural", name: "Richard" },
|
||
{ id: "en-US-TinaNeural", name: "Tina" },
|
||
{ id: "en-US-SteffanNeural", name: "Steffan" },
|
||
];
|
||
|
||
const HISTORY_KEY = "voice_history";
|
||
const HISTORY_MAX = 50;
|
||
const MODE_KEY = "hope_conversation_mode";
|
||
const MODEL_KEY = "preferred_model";
|
||
|
||
// ── State ───────────────────────────────────────────────────────────────────
|
||
let room = null;
|
||
let agentParticipant = null;
|
||
let agentReady = false;
|
||
let uiState = "idle"; // idle | connecting | listening | thinking | speaking
|
||
let micLevel = 0; // 0..1, smoothed
|
||
let agentAudioLevel = 0; // 0..1, smoothed
|
||
let userSpeaking = false; // local VAD-ish flag from mic level
|
||
let wakeLock = null;
|
||
let conversationMode = localStorage.getItem(MODE_KEY) === "text" ? "text" : "voice";
|
||
let connectionPromise = null;
|
||
|
||
// ── Model (brain) state — populated from the agent's data channel ──────────
|
||
let availableModels = []; // ids published by the agent
|
||
let selectedModel = localStorage.getItem(MODEL_KEY) || null;
|
||
|
||
function escapeHtml(s) {
|
||
const d = document.createElement("div");
|
||
d.textContent = s || "";
|
||
return d.innerHTML;
|
||
}
|
||
|
||
// ── DOM ─────────────────────────────────────────────────────────────────────
|
||
const startBtn = document.getElementById("startBtn");
|
||
const stopBtn = document.getElementById("stopBtn");
|
||
const muteBtn = document.getElementById("muteBtn");
|
||
let micMuted = false;
|
||
const statusEl = document.getElementById("status");
|
||
const messagesEl = document.getElementById("messages");
|
||
const ledWallEl = document.getElementById("ledWall");
|
||
const stateLabelEl = document.getElementById("stateLabel");
|
||
const thinkingChipEl = document.getElementById("thinkingChip");
|
||
const bargeHintEl = document.getElementById("bargeHint");
|
||
const micMeterFillEl = document.getElementById("micMeterFill");
|
||
const settingsBtn = document.getElementById("settingsBtn");
|
||
const settingsSheet = document.getElementById("settingsSheet");
|
||
const sheetBackdrop = document.getElementById("sheetBackdrop");
|
||
const closeSheetBtn = document.getElementById("closeSheetBtn");
|
||
const voiceListEl = document.getElementById("voiceList");
|
||
const modelListEl = document.getElementById("modelList");
|
||
const modelNoteEl = document.getElementById("modelNote");
|
||
const clearBtn = document.getElementById("clearBtn");
|
||
const tasksBtn = document.getElementById("tasksBtn");
|
||
const tasksPanel = document.getElementById("tasksPanel");
|
||
const tasksList = document.getElementById("tasksList");
|
||
const tasksBadge = document.getElementById("tasksBadge");
|
||
const closeTasksBtn = document.getElementById("closeTasksBtn");
|
||
const voiceModeBtn = document.getElementById("voiceModeBtn");
|
||
const textModeBtn = document.getElementById("textModeBtn");
|
||
const modeDescriptionEl = document.getElementById("modeDescription");
|
||
const transcriptModeLabelEl = document.getElementById("transcriptModeLabel");
|
||
const chatComposer = document.getElementById("chatComposer");
|
||
const chatInput = document.getElementById("chatInput");
|
||
const sendChatBtn = document.getElementById("sendChatBtn");
|
||
|
||
// ── Background tasks state ──────────────────────────────────────────────────
|
||
const tasks = new Map(); // task_id -> {id, description, status, steps: [], result, error}
|
||
|
||
function updateTasksBadge() {
|
||
const active = [...tasks.values()].filter(t => t.status === "running" || t.status === "pending").length;
|
||
if (active > 0) {
|
||
tasksBadge.hidden = false;
|
||
tasksBadge.textContent = String(active);
|
||
} else {
|
||
tasksBadge.hidden = true;
|
||
}
|
||
}
|
||
|
||
function renderTasks() {
|
||
const items = [...tasks.values()].sort((a, b) => (b.created_at || 0) - (a.created_at || 0));
|
||
if (items.length === 0) {
|
||
tasksList.innerHTML = '<div class="tasks-empty"><strong>Nothing running</strong><span>Ask Hope to research or compare something.</span></div>';
|
||
return;
|
||
}
|
||
tasksList.innerHTML = "";
|
||
for (const t of items) {
|
||
const card = document.createElement("div");
|
||
card.className = "task-card";
|
||
const statusClass = t.status || "pending";
|
||
card.innerHTML = `
|
||
<div class="task-card-header">
|
||
<span class="task-status-dot ${statusClass}"></span>
|
||
<span class="task-desc">${escapeHtml(t.description)}</span>
|
||
</div>
|
||
<ul class="task-steps">${t.steps.map(s => `<li class="task-step ${s.role}">${escapeHtml(s.content)}</li>`).join("")}</ul>
|
||
${t.result ? `<div class="task-result">${escapeHtml(t.result)}</div>` : ""}
|
||
${t.error ? `<div class="task-error">${escapeHtml(t.error)}</div>` : ""}
|
||
`;
|
||
tasksList.appendChild(card);
|
||
}
|
||
}
|
||
|
||
function handleTaskEvent(data) {
|
||
const { task_id, event, role, content, result, error, task } = data;
|
||
if (event === "created" && task) {
|
||
tasks.set(task_id, { ...task, steps: [] });
|
||
} else if (event === "step") {
|
||
const t = tasks.get(task_id);
|
||
if (t) {
|
||
t.steps.push({ role, content });
|
||
t.status = "running";
|
||
}
|
||
} else if (event === "completed") {
|
||
const t = tasks.get(task_id);
|
||
if (t) { t.status = "completed"; t.result = result; }
|
||
} else if (event === "failed") {
|
||
const t = tasks.get(task_id);
|
||
if (t) { t.status = "failed"; t.error = error; }
|
||
}
|
||
updateTasksBadge();
|
||
renderTasks();
|
||
}
|
||
|
||
function setTasksOpen(open) {
|
||
tasksPanel.classList.toggle("open", open);
|
||
tasksBtn.setAttribute("aria-expanded", String(open));
|
||
}
|
||
|
||
tasksBtn.setAttribute("aria-expanded", "false");
|
||
tasksBtn.addEventListener("click", () => setTasksOpen(!tasksPanel.classList.contains("open")));
|
||
closeTasksBtn.addEventListener("click", () => setTasksOpen(false));
|
||
|
||
// ── Voice / text mode ──────────────────────────────────────────────────────
|
||
function updateModeUI() {
|
||
const isText = conversationMode === "text";
|
||
document.body.dataset.mode = conversationMode;
|
||
voiceModeBtn.classList.toggle("active", !isText);
|
||
textModeBtn.classList.toggle("active", isText);
|
||
voiceModeBtn.setAttribute("aria-pressed", String(!isText));
|
||
textModeBtn.setAttribute("aria-pressed", String(isText));
|
||
modeDescriptionEl.textContent = isText
|
||
? "Type when quiet is better. Switch back to voice without losing the thread."
|
||
: "Speak naturally. I’ll listen, remember, and answer in real time.";
|
||
transcriptModeLabelEl.textContent = isText ? "Shared voice and text history" : "Live voice transcript";
|
||
startBtn.innerHTML = isText
|
||
? '<span class="button-spark" aria-hidden="true"></span>Connect to Hope'
|
||
: '<span class="button-spark" aria-hidden="true"></span>Start conversation';
|
||
if (isText) {
|
||
setState(room ? "listening" : "idle");
|
||
window.setTimeout(() => chatInput.focus(), 0);
|
||
} else {
|
||
// Refresh mode-specific copy even when the underlying state is unchanged.
|
||
setState(uiState);
|
||
}
|
||
}
|
||
|
||
async function setConversationMode(mode) {
|
||
if (mode !== "voice" && mode !== "text") return;
|
||
const previousMode = conversationMode;
|
||
conversationMode = mode;
|
||
localStorage.setItem(MODE_KEY, mode);
|
||
updateModeUI();
|
||
if (mode === "voice" && previousMode === "text") {
|
||
chatInput.blur();
|
||
window.setTimeout(() => {
|
||
document.querySelector(".workspace").scrollIntoView({ block: "start", behavior: "smooth" });
|
||
}, 0);
|
||
}
|
||
|
||
if (!room || !room.localParticipant) return;
|
||
sendModeMessage(mode);
|
||
if (mode === "text") {
|
||
await room.localParticipant.setMicrophoneEnabled(false);
|
||
micMuted = true;
|
||
muteBtn.disabled = true;
|
||
updateMuteIcon();
|
||
stopVisualLoop();
|
||
statusEl.textContent = "Secure room · Text mode";
|
||
} else {
|
||
micMuted = false;
|
||
muteBtn.disabled = false;
|
||
updateMuteIcon();
|
||
startVisualLoop();
|
||
try {
|
||
await room.startAudio();
|
||
await room.localParticipant.setMicrophoneEnabled(true);
|
||
setupMicAnalyser();
|
||
statusEl.textContent = "Secure room · Microphone on";
|
||
} catch (err) {
|
||
// Keep the conversation usable even if this browser denies or does
|
||
// not implement microphone capture. The mute control can retry.
|
||
console.warn("Microphone could not be enabled:", err);
|
||
micMuted = true;
|
||
updateMuteIcon();
|
||
statusEl.textContent = "Voice mode · Microphone unavailable";
|
||
}
|
||
}
|
||
}
|
||
|
||
voiceModeBtn.addEventListener("click", () => setConversationMode("voice").catch(console.error));
|
||
textModeBtn.addEventListener("click", () => setConversationMode("text").catch(console.error));
|
||
|
||
// ── Audio analysers (mic + remote agent audio) ─────────────────────────────
|
||
let micCtx = null;
|
||
let micAnalyser = null;
|
||
let micBuf = null;
|
||
let remoteCtx = null;
|
||
let remoteAnalyser = null;
|
||
let remoteFreqBuf = null;
|
||
|
||
function setupMicAnalyser() {
|
||
try {
|
||
const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
|
||
const mst = pub && pub.track && pub.track.mediaStreamTrack;
|
||
if (mst) {
|
||
micCtx = new AudioContext();
|
||
micAnalyser = micCtx.createAnalyser();
|
||
micAnalyser.fftSize = 512;
|
||
micBuf = new Float32Array(micAnalyser.fftSize);
|
||
micCtx.createMediaStreamSource(new MediaStream([mst])).connect(micAnalyser);
|
||
}
|
||
} catch (e) { /* fall back to SDK audioLevel in the meter loop */ }
|
||
}
|
||
|
||
function setupRemoteAudioAnalyser(audioEl) {
|
||
try {
|
||
if (!audioEl.srcObject) return;
|
||
remoteCtx = new AudioContext();
|
||
remoteAnalyser = remoteCtx.createAnalyser();
|
||
remoteAnalyser.fftSize = 256;
|
||
remoteFreqBuf = new Uint8Array(remoteAnalyser.frequencyBinCount);
|
||
remoteCtx.createMediaStreamSource(audioEl.srcObject).connect(remoteAnalyser);
|
||
} catch (e) { /* analyser optional */ }
|
||
}
|
||
|
||
function teardownAudio() {
|
||
for (const ctx of [micCtx, remoteCtx]) {
|
||
if (ctx && ctx.state !== "closed") ctx.close().catch(() => {});
|
||
}
|
||
micCtx = micAnalyser = micBuf = null;
|
||
remoteCtx = remoteAnalyser = remoteFreqBuf = null;
|
||
}
|
||
|
||
// ── LED wall — a grid of dots that pulses with the agent's audio ────────────
|
||
const LED_COLS = 21;
|
||
const LED_ROWS = 21;
|
||
const LED_GROW = 1.4; // how much the dots grow toward the center
|
||
let ledDots = [];
|
||
let ledWallTime = 0;
|
||
|
||
function buildLedWall() {
|
||
const inner = Math.floor(LED_COLS / 2);
|
||
const frag = document.createDocumentFragment();
|
||
for (let r = 0; r < LED_ROWS; r++) {
|
||
for (let c = 0; c < LED_COLS; c++) {
|
||
const b = document.createElement("button");
|
||
b.type = "button";
|
||
b.className = "led-dot";
|
||
b.disabled = true;
|
||
b.tabIndex = -1;
|
||
// Radial position: 0 at the center, 1 at the corner
|
||
const dx = (c - inner) / (LED_COLS / 2);
|
||
const dy = (r - inner) / (LED_ROWS / 2);
|
||
const rad = Math.sqrt(dx * dx + dy * dy);
|
||
// Push a radial cluster toward the center; scale its base brightness down
|
||
// as it moves outward so the wall reads as a soft round bloom.
|
||
b.dataset.rad = rad.toFixed(4);
|
||
// A per-dot phase so the wave ripples rather than flashing in unison.
|
||
b.dataset.phase = (Math.sin(r * 12.9898 + c * 78.233) * 43758.5453 % 1 * Math.PI * 2).toFixed(3);
|
||
ledDots.push(b);
|
||
frag.appendChild(b);
|
||
}
|
||
}
|
||
ledWallEl.appendChild(frag);
|
||
}
|
||
|
||
// Renders one frame of the LED wall. `level` is 0..1 (agent audio when speaking,
|
||
// mic when listening). `mode` picks the palette/gradient by conversation state.
|
||
function renderLedWall(level, mode) {
|
||
ledWallTime += 1;
|
||
ledWallEl.dataset.mode = mode;
|
||
const n = ledDots.length;
|
||
for (let i = 0; i < n; i++) {
|
||
const dot = ledDots[i];
|
||
const rad = parseFloat(dot.dataset.rad);
|
||
const phase = parseFloat(dot.dataset.phase || 0);
|
||
|
||
let vis = 0;
|
||
let size = 1;
|
||
|
||
if (mode === "idle") {
|
||
// A faint, static dot matrix — present but clearly dormant.
|
||
vis = 0.16 - rad * 0.08;
|
||
size = 1;
|
||
} else if (level > 0.005) {
|
||
// Centered breathing bloom that widens and brightens with the
|
||
// signal. A soft gaussian falloff keeps it circular; a gentle
|
||
// per-dot phase ripple adds life without banding.
|
||
const lvl = Math.min(1, level * 1.3);
|
||
const falloff = Math.exp(-rad * rad * (2.0 - lvl * 1.1));
|
||
const ripple = 0.62 + 0.38 * Math.sin(ledWallTime * 0.32 + phase);
|
||
vis = falloff * (0.35 + lvl) * ripple;
|
||
size = 1 + (1 - rad) * LED_GROW * lvl;
|
||
// Keep listening dimmer than speaking so the wall leans into the
|
||
// agent's output.
|
||
if (mode !== "speaking") vis *= 0.55;
|
||
} else {
|
||
// Powered state with no signal (e.g. thinking gap) — soft breathing.
|
||
const breathe = 0.5 + 0.5 * Math.sin(ledWallTime * 0.12 + phase);
|
||
vis = 0.14 - rad * 0.06 + 0.04 * breathe;
|
||
size = 1 + (1 - rad) * 0.3;
|
||
}
|
||
|
||
vis = Math.max(0.02, Math.min(1, vis));
|
||
// Dot grows toward the center and sits on a subtle 4px grid inset.
|
||
const px = (2 * size).toFixed(1);
|
||
dot.style.setProperty("--sz", `${px}px`);
|
||
dot.style.opacity = vis.toFixed(3);
|
||
}
|
||
}
|
||
|
||
function readMicLevel() {
|
||
let level = 0;
|
||
if (micAnalyser && micBuf) {
|
||
micAnalyser.getFloatTimeDomainData(micBuf);
|
||
for (let i = 0; i < micBuf.length; i++) level = Math.max(level, Math.abs(micBuf[i]));
|
||
} else if (room && room.localParticipant) {
|
||
level = room.localParticipant.audioLevel || 0;
|
||
}
|
||
// Smooth so the orb pulses rather than flickers
|
||
micLevel = micLevel * 0.6 + level * 0.4;
|
||
return micLevel;
|
||
}
|
||
|
||
function readAgentAudioLevel() {
|
||
let level = 0;
|
||
if (remoteAnalyser && remoteFreqBuf) {
|
||
remoteAnalyser.getByteFrequencyData(remoteFreqBuf);
|
||
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
||
level /= 255;
|
||
} else if (agentParticipant && agentParticipant.audioLevel !== undefined) {
|
||
level = agentParticipant.audioLevel || 0;
|
||
}
|
||
agentAudioLevel = agentAudioLevel * 0.6 + level * 0.4;
|
||
return agentAudioLevel;
|
||
}
|
||
|
||
// ── State machine ───────────────────────────────────────────────────────────
|
||
const STATE_LABELS = {
|
||
voice: {
|
||
idle: "Ready when you are",
|
||
connecting: "Opening a secure room",
|
||
listening: "Listening",
|
||
thinking: "Thinking it through",
|
||
speaking: "Hope is speaking",
|
||
},
|
||
text: {
|
||
idle: "Ready to type",
|
||
connecting: "Opening a secure room",
|
||
listening: "Text chat ready",
|
||
thinking: "Hope is writing",
|
||
speaking: "Hope replied",
|
||
},
|
||
};
|
||
|
||
function setState(next) {
|
||
uiState = next;
|
||
ledWallEl.dataset.state = next;
|
||
stateLabelEl.textContent = STATE_LABELS[conversationMode][next] || next;
|
||
|
||
// Thinking timer: show the "thinking..." chip if it lingers past 2s
|
||
if (next === "thinking") {
|
||
thinkingChipEl.hidden = true;
|
||
setTimeout(() => {
|
||
if (uiState === "thinking") thinkingChipEl.hidden = false;
|
||
}, 2000);
|
||
} else {
|
||
thinkingChipEl.hidden = true;
|
||
}
|
||
|
||
// Barge-in hint only while the agent is speaking
|
||
bargeHintEl.hidden = next !== "speaking";
|
||
|
||
if (next === "idle") {
|
||
micLevel = 0;
|
||
agentAudioLevel = 0;
|
||
userSpeaking = false;
|
||
}
|
||
}
|
||
|
||
updateModeUI();
|
||
|
||
function inferState() {
|
||
if (!room || !room.localParticipant) return "idle";
|
||
if (!agentParticipant) return "connecting";
|
||
readMicLevel();
|
||
readAgentAudioLevel();
|
||
userSpeaking = micLevel > 0.035;
|
||
if (agentAudioLevel > 0.02) return "speaking";
|
||
if (userSpeaking) return "listening";
|
||
// Once agent audio ends, return to listening. The previous implementation
|
||
// converted speaking → thinking forever, leaving the UI apparently frozen.
|
||
if (uiState === "speaking") return "listening";
|
||
if (uiState === "thinking") return "thinking";
|
||
return "listening";
|
||
}
|
||
|
||
// ── Visual loop: drives orb + meter + state inference ───────────────────────
|
||
let visualTimer = null;
|
||
function startVisualLoop() {
|
||
if (visualTimer) clearInterval(visualTimer);
|
||
visualTimer = setInterval(() => {
|
||
if (!room || !room.localParticipant) return;
|
||
setState(inferState());
|
||
|
||
// Mic meter (thin bar under the status line)
|
||
micMeterFillEl.style.width = `${Math.min(100, Math.round(micLevel * 250))}%`;
|
||
|
||
// LED wall: pulse on the agent's rendered audio when speaking, the mic
|
||
// while listening, and hold a faint matrix when idle.
|
||
if (uiState === "speaking") {
|
||
renderLedWall(agentAudioLevel, "speaking");
|
||
} else if (uiState === "listening" && userSpeaking) {
|
||
renderLedWall(micLevel, "listening");
|
||
} else {
|
||
renderLedWall(0, "idle");
|
||
}
|
||
|
||
// Agent audio visualizer bars on the latest agent bubble
|
||
const viz = messagesEl.querySelector(".message-row.agent:last-of-type .audio-viz");
|
||
if (viz) {
|
||
let level = 0;
|
||
if (remoteAnalyser && remoteFreqBuf) {
|
||
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
||
level /= 255;
|
||
}
|
||
const bars = viz.children;
|
||
for (let i = 0; i < bars.length; i++) {
|
||
// Per-bar phase offset so the bars dance rather than move in lockstep
|
||
const h = Math.max(0.15, level * (0.6 + 0.4 * Math.sin(Date.now() / 120 + i * 1.3)));
|
||
bars[i].style.height = `${h * 100}%`;
|
||
}
|
||
}
|
||
}, 80);
|
||
}
|
||
|
||
function stopVisualLoop() {
|
||
if (visualTimer) clearInterval(visualTimer);
|
||
visualTimer = null;
|
||
micMeterFillEl.style.width = "0%";
|
||
renderLedWall(0, "idle");
|
||
}
|
||
|
||
// ── Conversation history (localStorage) ─────────────────────────────────────
|
||
function loadHistory() {
|
||
try {
|
||
const raw = localStorage.getItem(HISTORY_KEY);
|
||
if (!raw) return [];
|
||
const arr = JSON.parse(raw);
|
||
return Array.isArray(arr) ? arr.slice(-HISTORY_MAX) : [];
|
||
} catch (e) { return []; }
|
||
}
|
||
|
||
function persistHistory() {
|
||
try {
|
||
localStorage.setItem(HISTORY_KEY, JSON.stringify(history.slice(-HISTORY_MAX)));
|
||
} catch (e) { /* storage full or unavailable */ }
|
||
}
|
||
|
||
let history = loadHistory();
|
||
|
||
const DELIVERY_MARKERS = /\[(?:appreciative|calm|concerned|curious|encouraging|excited|happy|joking|quiet|reassuring|reflective|sad|serious|surprised|sympathetic|urgent|whispering|laughter|sighing|breathing|neutral)\]\s*/gi;
|
||
|
||
function cleanSpokenText(text) {
|
||
return String(text || "").replace(DELIVERY_MARKERS, "").trim();
|
||
}
|
||
|
||
function formatTimestamp(ts) {
|
||
if (!ts) return "";
|
||
return new Date(ts).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" });
|
||
}
|
||
|
||
function renderMessage(msg) {
|
||
const wrap = document.createElement("div");
|
||
wrap.className = `message-row ${msg.role}`;
|
||
wrap.dataset.ts = formatTimestamp(msg.ts);
|
||
|
||
if (msg.role === "agent") {
|
||
const dot = document.createElement("span");
|
||
dot.className = "avatar-dot";
|
||
wrap.appendChild(dot);
|
||
}
|
||
|
||
const bubble = document.createElement("div");
|
||
bubble.className = "bubble";
|
||
bubble.textContent = cleanSpokenText(msg.text);
|
||
if (msg.role === "agent") {
|
||
const viz = document.createElement("span");
|
||
viz.className = "audio-viz";
|
||
for (let i = 0; i < 5; i++) viz.appendChild(document.createElement("i"));
|
||
bubble.appendChild(viz);
|
||
}
|
||
wrap.appendChild(bubble);
|
||
messagesEl.appendChild(wrap);
|
||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||
}
|
||
|
||
function addMessage(role, text) {
|
||
clearPartials();
|
||
const msg = { role, text, ts: Date.now() };
|
||
history.push(msg);
|
||
persistHistory();
|
||
renderMessage(msg);
|
||
return msg;
|
||
}
|
||
|
||
// Partial transcripts render greyed and update in place until a final arrives.
|
||
function addPartial(role, text) {
|
||
const rows = messagesEl.querySelectorAll(".message-row.partial");
|
||
let last = null;
|
||
for (const r of rows) last = r;
|
||
if (last && last.dataset.role === role) {
|
||
last.querySelector(".bubble").textContent = cleanSpokenText(text);
|
||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||
return;
|
||
}
|
||
const wrap = document.createElement("div");
|
||
wrap.className = `message-row ${role} partial`;
|
||
wrap.dataset.role = role;
|
||
if (role === "agent") {
|
||
const dot = document.createElement("span");
|
||
dot.className = "avatar-dot";
|
||
wrap.appendChild(dot);
|
||
}
|
||
const bubble = document.createElement("div");
|
||
bubble.className = "bubble partial";
|
||
bubble.textContent = cleanSpokenText(text);
|
||
wrap.appendChild(bubble);
|
||
messagesEl.appendChild(wrap);
|
||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||
}
|
||
|
||
function clearPartials() {
|
||
messagesEl.querySelectorAll(".message-row.partial").forEach((el) => el.remove());
|
||
}
|
||
|
||
function renderHistory() {
|
||
messagesEl.innerHTML = "";
|
||
for (const msg of history) renderMessage(msg);
|
||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||
}
|
||
|
||
clearBtn.addEventListener("click", () => {
|
||
history = [];
|
||
persistHistory();
|
||
clearPartials();
|
||
renderHistory();
|
||
});
|
||
|
||
renderHistory();
|
||
|
||
// ── Transcripts ─────────────────────────────────────────────────────────────
|
||
// Primary: livekit-agents built-in text stream (word-by-word when available).
|
||
function registerTranscriptionStream(rm) {
|
||
try {
|
||
rm.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => {
|
||
const text = await reader.readAll();
|
||
if (!text || !text.trim()) return;
|
||
const attrs = (reader.info && reader.info.attributes) || {};
|
||
const trackId = attrs["lk.transcribed_track_id"] || "";
|
||
// Distinguish user vs agent by the transcribed track's owner.
|
||
let role = "agent";
|
||
if (participantInfo && participantInfo.identity === AGENT_NAME) {
|
||
role = "agent";
|
||
} else if (trackId) {
|
||
for (const p of rm.remoteParticipants.values()) {
|
||
const pub = p.getTrackPublication(trackId);
|
||
if (pub) { role = isAgent(p) ? "agent" : "user"; break; }
|
||
}
|
||
}
|
||
// Use partial rendering: updates the same row in place until final.
|
||
addPartial(role, text.trim());
|
||
});
|
||
} catch (e) {
|
||
// Older SDKs may not support registerTextStreamHandler — the data
|
||
// channel handler below still covers transcripts.
|
||
console.warn("text stream registration failed:", e);
|
||
}
|
||
}
|
||
|
||
// Secondary: agent publishes {type:"transcript"} on topic "transcript".
|
||
function handleDataPacket(payload, participant, topic) {
|
||
try {
|
||
const msg = JSON.parse(new TextDecoder().decode(payload));
|
||
if (msg.type === "agent_ready") {
|
||
agentReady = true;
|
||
} else if (msg.type === "transcript") {
|
||
addMessage(msg.role || "agent", msg.text);
|
||
if ((msg.role || "agent") === "agent") {
|
||
if (conversationMode === "text") {
|
||
setState("listening");
|
||
chatInput.focus();
|
||
} else if (uiState === "thinking") {
|
||
// Text arrives even when headless playback or an analyser is
|
||
// unavailable; use it to release the thinking state.
|
||
setState("speaking");
|
||
}
|
||
}
|
||
} else if (msg.type === "task_event") {
|
||
handleTaskEvent(msg);
|
||
} else if (msg.type === "models") {
|
||
availableModels = Array.isArray(msg.models) ? msg.models : [];
|
||
// Don't clobber a saved preference — the agent acks the applied
|
||
// model separately via model_changed.
|
||
if (!selectedModel && msg.selected) selectedModel = msg.selected;
|
||
renderModelList();
|
||
} else if (msg.type === "model_changed") {
|
||
if (msg.model) {
|
||
selectedModel = msg.model;
|
||
localStorage.setItem(MODEL_KEY, selectedModel);
|
||
renderModelList();
|
||
}
|
||
}
|
||
// set_voice/set_model messages flow the other way; nothing else to do client-side.
|
||
} catch (e) {
|
||
console.warn("bad data packet", e);
|
||
}
|
||
}
|
||
|
||
// ── Voice settings sheet ────────────────────────────────────────────────────
|
||
function currentVoiceId() {
|
||
const saved = localStorage.getItem("preferred_voice");
|
||
return VOICES.some((v) => v.id === saved) ? saved : VOICES[0].id;
|
||
}
|
||
|
||
function renderVoiceList() {
|
||
voiceListEl.innerHTML = "";
|
||
const active = currentVoiceId();
|
||
for (const v of VOICES) {
|
||
const card = document.createElement("div");
|
||
card.className = "voice-card" + (v.id === active ? " active" : "");
|
||
card.dataset.voice = v.id;
|
||
|
||
const name = document.createElement("span");
|
||
name.className = "voice-name";
|
||
name.textContent = v.name + (v.tag ? ` (${v.tag})` : "");
|
||
|
||
const selected = v.id === active;
|
||
card.setAttribute("role", "radio");
|
||
card.setAttribute("aria-checked", String(selected));
|
||
card.tabIndex = 0;
|
||
card.appendChild(name);
|
||
card.addEventListener("click", () => selectVoice(v.id));
|
||
card.addEventListener("keydown", (event) => {
|
||
if (event.key === "Enter" || event.key === " ") {
|
||
event.preventDefault();
|
||
selectVoice(v.id);
|
||
}
|
||
});
|
||
voiceListEl.appendChild(card);
|
||
}
|
||
}
|
||
|
||
function selectVoice(voiceId) {
|
||
localStorage.setItem("preferred_voice", voiceId);
|
||
renderVoiceList();
|
||
if (room && agentParticipant) {
|
||
sendVoiceMessage(voiceId);
|
||
const v = VOICES.find((x) => x.id === voiceId);
|
||
addMessage("system", `Voice changed to ${v ? v.name : voiceId}`);
|
||
}
|
||
}
|
||
|
||
// ── Model (brain) settings ──────────────────────────────────────────────────
|
||
function currentModelId() {
|
||
if (availableModels.length === 0) return selectedModel;
|
||
if (selectedModel && availableModels.includes(selectedModel)) return selectedModel;
|
||
return availableModels[0];
|
||
}
|
||
|
||
function renderModelList() {
|
||
modelListEl.innerHTML = "";
|
||
if (!availableModels.length) {
|
||
modelNoteEl.hidden = false;
|
||
modelNoteEl.textContent = room
|
||
? "Loading available models…"
|
||
: "Start a conversation to load the model list.";
|
||
return;
|
||
}
|
||
modelNoteEl.hidden = true;
|
||
const active = currentModelId();
|
||
for (const id of availableModels) {
|
||
const card = document.createElement("div");
|
||
card.className = "voice-card" + (id === active ? " active" : "");
|
||
card.dataset.model = id;
|
||
card.setAttribute("role", "radio");
|
||
card.setAttribute("aria-checked", String(id === active));
|
||
card.tabIndex = 0;
|
||
|
||
const name = document.createElement("span");
|
||
name.className = "voice-name";
|
||
name.textContent = id;
|
||
|
||
card.appendChild(name);
|
||
card.addEventListener("click", () => selectModel(id));
|
||
card.addEventListener("keydown", (event) => {
|
||
if (event.key === "Enter" || event.key === " ") {
|
||
event.preventDefault();
|
||
selectModel(id);
|
||
}
|
||
});
|
||
modelListEl.appendChild(card);
|
||
}
|
||
}
|
||
|
||
function selectModel(modelId) {
|
||
selectedModel = modelId;
|
||
localStorage.setItem(MODEL_KEY, modelId);
|
||
renderModelList();
|
||
if (room && agentParticipant) {
|
||
sendModelMessage(modelId);
|
||
addMessage("system", `Brain changed to ${modelId}`);
|
||
}
|
||
}
|
||
|
||
function openSheet() {
|
||
renderVoiceList();
|
||
renderModelList();
|
||
sheetBackdrop.hidden = false;
|
||
settingsSheet.classList.add("open");
|
||
settingsBtn.setAttribute("aria-expanded", "true");
|
||
settingsSheet.querySelector(".voice-card")?.focus();
|
||
}
|
||
|
||
function closeSheet() {
|
||
settingsSheet.classList.remove("open");
|
||
settingsBtn.setAttribute("aria-expanded", "false");
|
||
window.setTimeout(() => { sheetBackdrop.hidden = true; }, 240);
|
||
settingsBtn.focus();
|
||
}
|
||
|
||
settingsBtn.setAttribute("aria-expanded", "false");
|
||
settingsBtn.addEventListener("click", openSheet);
|
||
closeSheetBtn.addEventListener("click", closeSheet);
|
||
sheetBackdrop.addEventListener("click", closeSheet);
|
||
document.addEventListener("keydown", (event) => {
|
||
if (event.key !== "Escape") return;
|
||
if (settingsSheet.classList.contains("open")) closeSheet();
|
||
if (tasksPanel.classList.contains("open")) setTasksOpen(false);
|
||
if (mediaPanel.classList.contains("open")) setMediaOpen(false);
|
||
});
|
||
// ── Wake lock ───────────────────────────────────────────────────────────────
|
||
async function acquireWakeLock() {
|
||
try {
|
||
if ("wakeLock" in navigator) {
|
||
wakeLock = await navigator.wakeLock.request("screen");
|
||
}
|
||
} catch (e) { /* not critical */ }
|
||
}
|
||
|
||
function releaseWakeLock() {
|
||
if (wakeLock) {
|
||
wakeLock.release().catch(() => {});
|
||
wakeLock = null;
|
||
}
|
||
}
|
||
|
||
// ── Agent detection ─────────────────────────────────────────────────────────
|
||
// The agent may already be in the room before we join (it persists between
|
||
// page loads), so scan remoteParticipants after connect AND watch for new
|
||
// joins — ParticipantConnected does not fire for pre-existing participants.
|
||
function isAgent(participant) {
|
||
// kind is the numeric proto enum; ParticipantKind.AGENT === 4.
|
||
// Prefer the SDK's own getter where available.
|
||
if (participant.isAgent !== undefined && participant.isAgent !== null) {
|
||
return participant.isAgent;
|
||
}
|
||
return participant.identity === AGENT_NAME
|
||
|| participant.kind === LivekitClient.ParticipantKind.AGENT;
|
||
}
|
||
|
||
function findAgent(rm) {
|
||
if (!rm) return null;
|
||
for (const p of rm.remoteParticipants.values()) {
|
||
if (isAgent(p)) return p;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function markAgentFound(rm, participant) {
|
||
agentParticipant = participant;
|
||
statusEl.textContent = conversationMode === "text"
|
||
? "Hope joined · Text chat ready"
|
||
: "Hope joined · Preparing microphone…";
|
||
// Synchronize voice, brain and interaction mode when the agent joins.
|
||
sendVoiceMessage(currentVoiceId(), rm);
|
||
if (selectedModel) sendModelMessage(selectedModel, rm);
|
||
sendModeMessage(conversationMode, rm);
|
||
}
|
||
|
||
function createRoom(token) {
|
||
const newRoom = new Room({
|
||
adaptiveStream: true,
|
||
dynacast: true,
|
||
});
|
||
|
||
// Listen for the agent joining after us
|
||
newRoom.on(RoomEvent.ParticipantConnected, (participant) => {
|
||
if (isAgent(participant)) markAgentFound(newRoom, participant);
|
||
});
|
||
|
||
// Attach and play incoming audio tracks (agent TTS)
|
||
newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
|
||
if (track.kind === LivekitClient.Track.Kind.Audio) {
|
||
const el = track.attach();
|
||
el.id = `audio-${participant.identity}`;
|
||
document.body.appendChild(el);
|
||
// Remote audio analyser drives the orb + visualizer while speaking
|
||
setupRemoteAudioAnalyser(el);
|
||
}
|
||
});
|
||
|
||
newRoom.on(RoomEvent.TrackUnsubscribed, (track) => {
|
||
track.detach().forEach((el) => el.remove());
|
||
});
|
||
|
||
// Browsers block audio playback until a user gesture; show an unlock
|
||
// button if the page loads without one.
|
||
newRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => {
|
||
if (!newRoom.canPlaybackAudio) {
|
||
const existing = document.getElementById("audioUnlockBtn");
|
||
if (!existing) {
|
||
const btn = document.createElement("button");
|
||
btn.id = "audioUnlockBtn";
|
||
btn.className = "btn btn-primary";
|
||
btn.textContent = "Tap to enable audio";
|
||
btn.style.cssText = "position:fixed;bottom:2rem;left:50%;transform:translateX(-50%);z-index:999;padding:1rem 2rem;font-size:1.1rem;border-radius:999px;box-shadow:0 4px 24px rgba(0,0,0,.5)";
|
||
btn.onclick = () => { newRoom.startAudio(); btn.remove(); };
|
||
document.body.appendChild(btn);
|
||
}
|
||
} else {
|
||
const existing = document.getElementById("audioUnlockBtn");
|
||
if (existing) existing.remove();
|
||
}
|
||
});
|
||
|
||
// Live text stream (word-by-word transcripts)
|
||
registerTranscriptionStream(newRoom);
|
||
|
||
// Data channel: transcripts (secondary source) + voice control
|
||
newRoom.on(RoomEvent.DataReceived, (payload, participant, kind, topic) => {
|
||
handleDataPacket(payload, participant, topic);
|
||
});
|
||
|
||
// Guard with `=== room` so a stale (retry-discarded) connection cannot
|
||
// clobber UI state of the active one.
|
||
newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
|
||
if (participant === newRoom.localParticipant && newRoom === room) {
|
||
statusEl.textContent = `Secure room · ${quality} connection`;
|
||
}
|
||
});
|
||
|
||
newRoom.on(RoomEvent.Disconnected, () => {
|
||
if (newRoom !== room) return;
|
||
teardownAudio();
|
||
stopVisualLoop();
|
||
releaseWakeLock();
|
||
statusEl.textContent = "Conversation ended";
|
||
setState("idle");
|
||
startBtn.disabled = false;
|
||
stopBtn.disabled = true;
|
||
agentParticipant = null;
|
||
agentReady = false;
|
||
// Leaving the conversation hands the room back to the music player.
|
||
resumePlaybackAfterTalk();
|
||
});
|
||
|
||
return newRoom.connect(LIVEKIT_URL, token, {
|
||
autoSubscribe: true,
|
||
}).then(() => {
|
||
// Pick up an agent that was already in the room before we joined
|
||
const existing = findAgent(newRoom);
|
||
if (existing) markAgentFound(newRoom, existing);
|
||
return newRoom;
|
||
});
|
||
}
|
||
|
||
function waitForAgentReady(timeoutMs) {
|
||
return new Promise((resolve) => {
|
||
if (agentReady) return resolve(true);
|
||
const deadline = Date.now() + timeoutMs;
|
||
const timer = setInterval(() => {
|
||
if (agentReady) {
|
||
clearInterval(timer);
|
||
resolve(true);
|
||
} else if (Date.now() > deadline) {
|
||
clearInterval(timer);
|
||
resolve(false);
|
||
}
|
||
}, 100);
|
||
});
|
||
}
|
||
|
||
function waitForAgent(timeoutMs) {
|
||
return new Promise((resolve) => {
|
||
if (findAgent(room)) return resolve(true);
|
||
const deadline = Date.now() + timeoutMs;
|
||
const timer = setInterval(() => {
|
||
const found = findAgent(room);
|
||
if (found) {
|
||
if (!agentParticipant) markAgentFound(room, found);
|
||
clearInterval(timer);
|
||
resolve(true);
|
||
} else if (Date.now() > deadline) {
|
||
clearInterval(timer);
|
||
resolve(false);
|
||
}
|
||
}, 250);
|
||
});
|
||
}
|
||
|
||
async function connectConversation() {
|
||
if (room && agentParticipant) return room;
|
||
if (connectionPromise) return connectionPromise;
|
||
|
||
connectionPromise = (async () => {
|
||
try {
|
||
setState("connecting");
|
||
statusEl.textContent = "Opening your secure room…";
|
||
await acquireWakeLock();
|
||
|
||
// Get a signed access token from the token endpoint.
|
||
const tokenData = await fetchToken(ROOM_NAME);
|
||
await enterMediaScope(tokenData.room);
|
||
room = await createRoom(tokenData.token);
|
||
|
||
// The agent is dispatched when we join. If it misses the dispatch
|
||
// (for example during a server restart), reconnect once.
|
||
statusEl.textContent = "Inviting Hope into the room…";
|
||
if (!(await waitForAgent(10000))) {
|
||
statusEl.textContent = "Hope is taking a moment · Retrying…";
|
||
const stale = room;
|
||
room = null;
|
||
agentParticipant = null;
|
||
agentReady = false;
|
||
await stale.disconnect();
|
||
const freshTokenData = await fetchToken(ROOM_NAME);
|
||
await enterMediaScope(freshTokenData.room);
|
||
room = await createRoom(freshTokenData.token);
|
||
if (!(await waitForAgent(15000))) {
|
||
throw new Error("Assistant did not join — reload the page and try again");
|
||
}
|
||
}
|
||
statusEl.textContent = "Hope joined · Finishing setup…";
|
||
if (!(await waitForAgentReady(10000))) {
|
||
throw new Error("Assistant joined but was not ready — try again");
|
||
}
|
||
|
||
sendModeMessage(conversationMode);
|
||
if (conversationMode === "voice") {
|
||
await room.startAudio();
|
||
statusEl.textContent = "Waiting for microphone permission…";
|
||
await room.localParticipant.setMicrophoneEnabled(true);
|
||
micMuted = false;
|
||
setupMicAnalyser();
|
||
startVisualLoop();
|
||
statusEl.textContent = "Secure room · Microphone on";
|
||
} else {
|
||
await room.localParticipant.setMicrophoneEnabled(false);
|
||
micMuted = true;
|
||
stopVisualLoop();
|
||
statusEl.textContent = "Secure room · Text mode";
|
||
}
|
||
|
||
startBtn.disabled = true;
|
||
stopBtn.disabled = false;
|
||
muteBtn.disabled = conversationMode !== "voice";
|
||
updateMuteIcon();
|
||
setState("listening");
|
||
return room;
|
||
} catch (err) {
|
||
console.error("Connection failed:", err);
|
||
statusEl.textContent = `Error: ${err.message}`;
|
||
setState("idle");
|
||
releaseWakeLock();
|
||
throw err;
|
||
} finally {
|
||
connectionPromise = null;
|
||
}
|
||
})();
|
||
|
||
return connectionPromise;
|
||
}
|
||
|
||
startBtn.addEventListener("click", () => {
|
||
connectConversation().catch(() => {});
|
||
});
|
||
|
||
// ── Text chat ──────────────────────────────────────────────────────────────
|
||
function resizeChatInput() {
|
||
chatInput.style.height = "auto";
|
||
chatInput.style.height = `${Math.min(chatInput.scrollHeight, 144)}px`;
|
||
}
|
||
|
||
chatInput.addEventListener("input", () => {
|
||
resizeChatInput();
|
||
sendChatBtn.disabled = !chatInput.value.trim();
|
||
});
|
||
|
||
chatInput.addEventListener("keydown", (event) => {
|
||
if (event.key === "Enter" && !event.shiftKey) {
|
||
event.preventDefault();
|
||
chatComposer.requestSubmit();
|
||
}
|
||
});
|
||
|
||
chatComposer.addEventListener("submit", async (event) => {
|
||
event.preventDefault();
|
||
const text = chatInput.value.trim();
|
||
if (!text) return;
|
||
|
||
sendChatBtn.disabled = true;
|
||
chatInput.disabled = true;
|
||
try {
|
||
if (conversationMode !== "text") await setConversationMode("text");
|
||
await connectConversation();
|
||
setState("thinking");
|
||
await room.localParticipant.sendText(text, { topic: "lk.chat" });
|
||
chatInput.value = "";
|
||
resizeChatInput();
|
||
} catch (err) {
|
||
console.error("Text message failed:", err);
|
||
statusEl.textContent = `Message not sent: ${err.message}`;
|
||
setState("listening");
|
||
} finally {
|
||
chatInput.disabled = false;
|
||
sendChatBtn.disabled = !chatInput.value.trim();
|
||
chatInput.focus();
|
||
}
|
||
});
|
||
|
||
// ── Stop ────────────────────────────────────────────────────────────────────
|
||
stopBtn.addEventListener("click", () => {
|
||
if (room) {
|
||
room.disconnect();
|
||
room = null;
|
||
agentParticipant = null;
|
||
agentReady = false;
|
||
}
|
||
teardownAudio();
|
||
stopVisualLoop();
|
||
releaseWakeLock();
|
||
document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove());
|
||
statusEl.textContent = "Conversation ended";
|
||
setState("idle");
|
||
startBtn.disabled = false;
|
||
stopBtn.disabled = true;
|
||
muteBtn.disabled = true;
|
||
micMuted = false;
|
||
updateMuteIcon();
|
||
// Ending the conversation resumes music that was paused for the talk.
|
||
resumePlaybackAfterTalk();
|
||
});
|
||
|
||
// ── Mic mute toggle ─────────────────────────────────────────────────────────
|
||
muteBtn.addEventListener("click", async () => {
|
||
if (!room || !room.localParticipant) return;
|
||
const nextMuted = !micMuted;
|
||
try {
|
||
await room.localParticipant.setMicrophoneEnabled(!nextMuted);
|
||
micMuted = nextMuted;
|
||
updateMuteIcon();
|
||
} catch (err) {
|
||
console.warn("Microphone toggle failed:", err);
|
||
statusEl.textContent = "Microphone unavailable in this browser";
|
||
}
|
||
});
|
||
|
||
function updateMuteIcon() {
|
||
const muted = muteBtn.classList.toggle("muted", micMuted);
|
||
muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone");
|
||
muteBtn.title = muted ? "Unmute microphone" : "Mute microphone";
|
||
}
|
||
|
||
// ── Helper: fetch a signed access token from the token endpoint ────────────
|
||
async function fetchToken(roomName) {
|
||
const resp = await fetch("/token", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ room: roomName }),
|
||
});
|
||
if (!resp.ok) throw new Error(`Token endpoint returned ${resp.status}`);
|
||
const data = await resp.json();
|
||
return { token: data.token, room: data.room || null };
|
||
}
|
||
|
||
// ── Helper: send voice selection to the agent via data channel ─────────────
|
||
function sendVoiceMessage(voice, targetRoom) {
|
||
const r = targetRoom || room;
|
||
if (!r || !r.localParticipant) return;
|
||
const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice }));
|
||
r.localParticipant.publishData(payload, {
|
||
topic: "voice-control",
|
||
reliable: true,
|
||
});
|
||
}
|
||
|
||
function sendModeMessage(mode, targetRoom) {
|
||
const r = targetRoom || room;
|
||
if (!r || !r.localParticipant) return;
|
||
const payload = new TextEncoder().encode(JSON.stringify({ type: "set_mode", mode }));
|
||
r.localParticipant.publishData(payload, {
|
||
topic: "conversation-control",
|
||
reliable: true,
|
||
});
|
||
}
|
||
|
||
// ── Helper: send model selection to the agent via data channel ─────────────
|
||
function sendModelMessage(model, targetRoom) {
|
||
const r = targetRoom || room;
|
||
if (!r || !r.localParticipant || !model) return;
|
||
const payload = new TextEncoder().encode(JSON.stringify({ type: "set_model", model }));
|
||
r.localParticipant.publishData(payload, {
|
||
topic: "voice-control",
|
||
reliable: true,
|
||
});
|
||
}
|
||
|
||
// ── Media player ────────────────────────────────────────────────────────────
|
||
// The browser <audio> element is the player. The media server owns library +
|
||
// per-room desired state; this UI reconciles against GET /media/api/state
|
||
// (polling) and reports real playback back. Hope drives the same state via
|
||
// her media tools, so voice commands and panel clicks stay in sync.
|
||
//
|
||
// Conversation rule (mirrored in agent.py): while a track plays the agent's
|
||
// mic is off. Connecting to Hope pauses music; ending the conversation
|
||
// resumes what was playing.
|
||
|
||
const mediaBtn = document.getElementById("mediaBtn");
|
||
const mediaPanel = document.getElementById("mediaPanel");
|
||
const closeMediaBtn = document.getElementById("closeMediaBtn");
|
||
const mediaBadge = document.getElementById("mediaBadge");
|
||
const mediaArt = document.getElementById("mediaArt");
|
||
const mediaArtPlaceholder = document.getElementById("mediaArtPlaceholder");
|
||
const mediaTitleEl = document.getElementById("mediaTitle");
|
||
const mediaChannelEl = document.getElementById("mediaChannel");
|
||
const mediaProgressWrap = document.getElementById("mediaProgressWrap");
|
||
const mediaProgressFill = document.getElementById("mediaProgressFill");
|
||
const mediaProgressLabel = document.getElementById("mediaProgressLabel");
|
||
const rewBtn = document.getElementById("rewBtn");
|
||
const fwdBtn = document.getElementById("fwdBtn");
|
||
const playPauseBtn = document.getElementById("playPauseBtn");
|
||
const playIcon = document.getElementById("playIcon");
|
||
const pauseIcon = document.getElementById("pauseIcon");
|
||
const stopMediaBtn = document.getElementById("stopMediaBtn");
|
||
const seekBar = document.getElementById("seekBar");
|
||
const mediaCurEl = document.getElementById("mediaCur");
|
||
const mediaDurEl = document.getElementById("mediaDur");
|
||
const volumeBar = document.getElementById("volumeBar");
|
||
const mediaSearchForm = document.getElementById("mediaSearchForm");
|
||
const mediaSearchInput = document.getElementById("mediaSearchInput");
|
||
const mediaResultsEl = document.getElementById("mediaResults");
|
||
const mediaChoicesWrap = document.getElementById("mediaChoicesWrap");
|
||
const mediaChoicesEl = document.getElementById("mediaChoices");
|
||
const mediaLibraryEl = document.getElementById("mediaLibrary");
|
||
const streamForm = document.getElementById("streamForm");
|
||
const streamNameInput = document.getElementById("streamName");
|
||
const streamUrlInput = document.getElementById("streamUrl");
|
||
const mediaAudio = document.getElementById("mediaAudio");
|
||
|
||
let mediaRoom = null; // LiveKit room name (player state scope)
|
||
let mediaCurrentTrackId = null; // track id loaded into the audio element
|
||
let mediaAppliedSeekTs = 0; // last applied seek_hint timestamp
|
||
let mediaWasPlayingBeforeTalk = false;
|
||
let mediaPollTimer = null;
|
||
let mediaLastReport = 0;
|
||
|
||
function mediaScope() {
|
||
return mediaRoom || "default";
|
||
}
|
||
|
||
async function mediaPost(path, body) {
|
||
const resp = await fetch(`/media/api${path}`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ room: mediaScope(), ...body }),
|
||
});
|
||
if (!resp.ok) {
|
||
const detail = await resp.json().catch(() => ({}));
|
||
throw new Error(detail.error || `media API ${path} -> ${resp.status}`);
|
||
}
|
||
return resp.json();
|
||
}
|
||
|
||
// Called just before joining the LiveKit room: carry whatever the audio
|
||
// element holds into the room-scoped player state, paused (talking pauses
|
||
// music). resumePlaybackAfterTalk() reverses this when the talk ends.
|
||
async function enterMediaScope(newRoom) {
|
||
if (!newRoom || newRoom === mediaRoom) return;
|
||
const wasPlaying = !!mediaCurrentTrackId && !mediaAudio.paused;
|
||
mediaWasPlayingBeforeTalk = wasPlaying;
|
||
const carriedTrack = mediaCurrentTrackId;
|
||
mediaRoom = newRoom;
|
||
if (carriedTrack) {
|
||
try {
|
||
await mediaPost("/play", { room: mediaRoom, track_id: carriedTrack, start_paused: true });
|
||
} catch (e) { console.warn("media scope transfer failed:", e); }
|
||
mediaAudio.pause();
|
||
}
|
||
}
|
||
|
||
async function resumePlaybackAfterTalk() {
|
||
if (!mediaWasPlayingBeforeTalk) return;
|
||
mediaWasPlayingBeforeTalk = false;
|
||
try { await mediaPost("/control", { action: "resume" }); } catch (e) { /* poll retries */ }
|
||
}
|
||
|
||
function publishMediaState(state) {
|
||
if (!room || !room.localParticipant) return;
|
||
const payload = new TextEncoder().encode(
|
||
JSON.stringify({ type: "media_state", state, track_id: mediaCurrentTrackId })
|
||
);
|
||
room.localParticipant.publishData(payload, { topic: "media-control", reliable: true });
|
||
}
|
||
|
||
function reportMediaPlayback(state) {
|
||
const body = {
|
||
state,
|
||
track_id: mediaCurrentTrackId,
|
||
position: isFinite(mediaAudio.currentTime) ? mediaAudio.currentTime : 0,
|
||
};
|
||
if (isFinite(mediaAudio.duration)) body.duration = mediaAudio.duration;
|
||
mediaPost("/report", body).catch(() => {});
|
||
}
|
||
|
||
// ── Polling + reconciliation ────────────────────────────────────────────────
|
||
function startMediaPolling() {
|
||
if (mediaPollTimer) return;
|
||
mediaPollTimer = setInterval(() => {
|
||
pollMediaState();
|
||
}, 1500);
|
||
}
|
||
|
||
async function pollMediaState() {
|
||
let s;
|
||
try {
|
||
const resp = await fetch(`/media/api/state?room=${encodeURIComponent(mediaScope())}`);
|
||
if (!resp.ok) return;
|
||
s = await resp.json();
|
||
} catch (e) {
|
||
return; // media server briefly unavailable; try again next tick
|
||
}
|
||
reconcileMedia(s);
|
||
}
|
||
|
||
function reconcileMedia(s) {
|
||
renderMediaChoices(s.choices || []);
|
||
updateMediaBadge(s);
|
||
|
||
const track = s.track;
|
||
const pendingJob = (s.pending || []).find(j => !track || j.track_id === track.id);
|
||
updateMediaProgress(track, pendingJob);
|
||
|
||
if (track && track.id !== mediaCurrentTrackId) {
|
||
loadMediaTrack(track);
|
||
} else if (track && track.audio_url && mediaAudio.getAttribute("src") !== track.audio_url) {
|
||
// The track was loaded before it had a stream (still downloading) and
|
||
// now has one — wire it up so playback can start without a reload.
|
||
mediaAudio.src = track.audio_url;
|
||
mediaAudio.load();
|
||
updateMediaNowCard(track);
|
||
}
|
||
if (!track && mediaCurrentTrackId && s.desired === "stopped") clearMediaAudio();
|
||
|
||
applyDesiredMediaState(s, track);
|
||
|
||
if (s.seek_hint && s.seek_hint.ts !== mediaAppliedSeekTs && isFinite(mediaAudio.duration)) {
|
||
mediaAppliedSeekTs = s.seek_hint.ts;
|
||
const target = Math.min(s.seek_hint.position, mediaAudio.duration);
|
||
if (Number.isFinite(target)) {
|
||
mediaAudio.currentTime = Math.max(0, target);
|
||
updateSeekUI();
|
||
}
|
||
}
|
||
}
|
||
|
||
function applyDesiredMediaState(s, track) {
|
||
if (s.desired === "playing") {
|
||
// audio_url presence means a playable stream exists — including a
|
||
// still-growing download, which plays progressively.
|
||
if (track && track.audio_url && mediaAudio.src && mediaAudio.paused) {
|
||
mediaAudio.play().catch((e) => console.warn("playback blocked:", e));
|
||
}
|
||
} else if (!mediaAudio.paused) {
|
||
mediaAudio.pause();
|
||
}
|
||
}
|
||
|
||
function loadMediaTrack(track) {
|
||
mediaCurrentTrackId = track.id;
|
||
updateMediaNowCard(track);
|
||
if (track.audio_url) {
|
||
if (mediaAudio.getAttribute("src") !== track.audio_url) {
|
||
mediaAudio.src = track.audio_url;
|
||
mediaAudio.load();
|
||
}
|
||
} else {
|
||
// Downloading or failed: no stream yet. reconcileMedia wires the
|
||
// stream URL in via polling as soon as one is available.
|
||
mediaAudio.removeAttribute("src");
|
||
mediaAudio.load();
|
||
updateMediaControls();
|
||
}
|
||
}
|
||
|
||
function clearMediaAudio() {
|
||
mediaCurrentTrackId = null;
|
||
mediaAudio.pause();
|
||
mediaAudio.removeAttribute("src");
|
||
mediaAudio.load();
|
||
mediaTitleEl.textContent = "Nothing loaded";
|
||
mediaChannelEl.textContent = "";
|
||
mediaArt.hidden = true;
|
||
mediaArtPlaceholder.hidden = false;
|
||
updateMediaControls();
|
||
}
|
||
|
||
function updateMediaNowCard(track) {
|
||
mediaTitleEl.textContent = track.title || track.id;
|
||
if (track.status === "failed") {
|
||
mediaChannelEl.textContent = "Download failed — pick another result or try again.";
|
||
} else if (track.status === "downloading") {
|
||
// Progressive streaming: the track plays while the download runs.
|
||
mediaChannelEl.textContent = track.channel
|
||
? `${track.channel} · downloading` : "Downloading…";
|
||
} else {
|
||
mediaChannelEl.textContent = track.channel || (track.kind === "stream" ? "Live stream" : "");
|
||
}
|
||
if (track.art_url) {
|
||
mediaArt.src = track.art_url;
|
||
mediaArt.hidden = false;
|
||
mediaArtPlaceholder.hidden = true;
|
||
} else {
|
||
mediaArt.hidden = true;
|
||
mediaArtPlaceholder.hidden = false;
|
||
}
|
||
}
|
||
|
||
function updateMediaProgress(track, job) {
|
||
if (!job) {
|
||
mediaProgressWrap.hidden = true;
|
||
return;
|
||
}
|
||
mediaProgressWrap.hidden = false;
|
||
const pct = Math.round(job.progress || 0);
|
||
mediaProgressFill.style.width = `${pct}%`;
|
||
const stage = job.stage === "queued" ? "Queued…" : "Downloading…";
|
||
mediaProgressLabel.textContent = `${stage} ${pct}%`;
|
||
if (track) updateMediaNowCard(track);
|
||
}
|
||
|
||
function updateMediaBadge(s) {
|
||
const pending = (s.pending || []).length;
|
||
const playing = !mediaAudio.paused && !!mediaCurrentTrackId;
|
||
if (pending > 0) {
|
||
mediaBadge.hidden = false;
|
||
mediaBadge.textContent = String(pending);
|
||
} else if (playing) {
|
||
mediaBadge.hidden = false;
|
||
mediaBadge.innerHTML = "♫";
|
||
} else {
|
||
mediaBadge.hidden = true;
|
||
}
|
||
}
|
||
|
||
// ── Playback controls ───────────────────────────────────────────────────────
|
||
function updateMediaControls() {
|
||
const playing = !mediaAudio.paused && !!mediaCurrentTrackId;
|
||
playPauseBtn.classList.toggle("playing", playing);
|
||
playPauseBtn.setAttribute("aria-label", playing ? "Pause" : "Play");
|
||
updateMediaBadge({ pending: [] });
|
||
}
|
||
|
||
function fmtTime(sec) {
|
||
if (!isFinite(sec) || sec < 0) return "0:00";
|
||
const s = Math.floor(sec % 60);
|
||
const m = Math.floor((sec / 60) % 60);
|
||
const h = Math.floor(sec / 3600);
|
||
return h ? `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}` : `${m}:${String(s).padStart(2, "0")}`;
|
||
}
|
||
|
||
function updateSeekUI() {
|
||
if (!isFinite(mediaAudio.duration) || mediaAudio.duration <= 0) {
|
||
seekBar.value = 0;
|
||
mediaDurEl.textContent = "0:00";
|
||
mediaCurEl.textContent = fmtTime(mediaAudio.currentTime);
|
||
return;
|
||
}
|
||
seekBar.max = Math.floor(mediaAudio.duration);
|
||
seekBar.value = Math.floor(mediaAudio.currentTime);
|
||
mediaCurEl.textContent = fmtTime(mediaAudio.currentTime);
|
||
mediaDurEl.textContent = fmtTime(mediaAudio.duration);
|
||
}
|
||
|
||
playPauseBtn.addEventListener("click", () => {
|
||
if (!mediaCurrentTrackId || !mediaAudio.src) return;
|
||
if (mediaAudio.paused) {
|
||
mediaAudio.play().catch((e) => console.warn("play failed:", e));
|
||
mediaPost("/control", { action: "resume" }).catch(() => {});
|
||
} else {
|
||
mediaAudio.pause();
|
||
mediaPost("/control", { action: "pause" }).catch(() => {});
|
||
}
|
||
});
|
||
|
||
rewBtn.addEventListener("click", () => {
|
||
if (!mediaAudio.src) return;
|
||
mediaAudio.currentTime = Math.max(0, mediaAudio.currentTime - 10);
|
||
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
|
||
});
|
||
|
||
fwdBtn.addEventListener("click", () => {
|
||
if (!mediaAudio.src || !isFinite(mediaAudio.duration)) return;
|
||
mediaAudio.currentTime = Math.min(mediaAudio.duration - 0.5, mediaAudio.currentTime + 10);
|
||
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
|
||
});
|
||
|
||
stopMediaBtn.addEventListener("click", () => {
|
||
clearMediaAudio();
|
||
mediaPost("/control", { action: "stop" }).catch(() => {});
|
||
publishMediaState("stopped");
|
||
});
|
||
|
||
seekBar.addEventListener("change", () => {
|
||
if (!mediaAudio.src || !isFinite(mediaAudio.duration)) return;
|
||
mediaAudio.currentTime = Number(seekBar.value);
|
||
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
|
||
});
|
||
|
||
volumeBar.addEventListener("input", () => {
|
||
mediaAudio.volume = Number(volumeBar.value) / 100;
|
||
try { localStorage.setItem("media_volume", volumeBar.value); } catch (e) { /* ok */ }
|
||
});
|
||
mediaAudio.volume = Number(localStorage.getItem("media_volume") ?? 90) / 100;
|
||
volumeBar.value = Math.round(mediaAudio.volume * 100);
|
||
|
||
mediaAudio.addEventListener("play", () => {
|
||
updateMediaControls();
|
||
reportMediaPlayback("playing");
|
||
publishMediaState("playing");
|
||
});
|
||
mediaAudio.addEventListener("pause", () => {
|
||
updateMediaControls();
|
||
if (mediaCurrentTrackId) {
|
||
reportMediaPlayback("paused");
|
||
publishMediaState("paused");
|
||
}
|
||
});
|
||
mediaAudio.addEventListener("ended", () => {
|
||
updateMediaControls();
|
||
reportMediaPlayback("stopped");
|
||
publishMediaState("stopped");
|
||
mediaPost("/control", { action: "stop" }).catch(() => {});
|
||
});
|
||
mediaAudio.addEventListener("timeupdate", () => {
|
||
updateSeekUI();
|
||
const now = Date.now();
|
||
if (!mediaAudio.paused && now - mediaLastReport > 5000) {
|
||
mediaLastReport = now;
|
||
reportMediaPlayback("playing");
|
||
}
|
||
});
|
||
mediaAudio.addEventListener("durationchange", updateSeekUI);
|
||
mediaAudio.addEventListener("error", () => {
|
||
// An empty src (track still downloading) also fires this — only surface
|
||
// real playback failures.
|
||
if (mediaCurrentTrackId && mediaAudio.getAttribute("src")) {
|
||
mediaChannelEl.textContent = "Playback error — the track may still be preparing.";
|
||
}
|
||
});
|
||
|
||
// ── Search / choices / library ──────────────────────────────────────────────
|
||
function fmtListDuration(sec) {
|
||
const s = Number(sec);
|
||
return isFinite(s) && s > 0 ? fmtTime(s) : "";
|
||
}
|
||
|
||
function mediaRowActions() {
|
||
const play = document.createElement("button");
|
||
play.className = "btn btn-secondary media-row-play";
|
||
play.type = "button";
|
||
play.textContent = "Play";
|
||
return play;
|
||
}
|
||
|
||
function renderMediaResults(results) {
|
||
mediaResultsEl.innerHTML = "";
|
||
if (!results || results.length === 0) {
|
||
mediaResultsEl.innerHTML = '<div class="media-empty">No results.</div>';
|
||
return;
|
||
}
|
||
for (const r of results) {
|
||
const row = document.createElement("div");
|
||
row.className = "media-row";
|
||
const info = document.createElement("div");
|
||
info.className = "media-row-info";
|
||
const dur = fmtListDuration(r.duration);
|
||
info.innerHTML = `<strong>${escapeHtml(r.title)}</strong>` +
|
||
`<span>${escapeHtml(r.channel || "")}${dur ? " · " + dur : ""}` +
|
||
`${r.in_library ? " · in library" : ""}</span>`;
|
||
const play = mediaRowActions();
|
||
play.addEventListener("click", () => playVideoId(r));
|
||
row.appendChild(info);
|
||
row.appendChild(play);
|
||
mediaResultsEl.appendChild(row);
|
||
}
|
||
}
|
||
|
||
async function playVideoId(r) {
|
||
try {
|
||
await mediaPost("/play", {
|
||
video_id: r.video_id,
|
||
title: r.title,
|
||
channel: r.channel || "",
|
||
});
|
||
} catch (e) {
|
||
mediaChannelEl.textContent = `Could not start: ${e.message}`;
|
||
}
|
||
}
|
||
|
||
mediaSearchForm.addEventListener("submit", async (event) => {
|
||
event.preventDefault();
|
||
const query = mediaSearchInput.value.trim();
|
||
if (!query) return;
|
||
mediaResultsEl.innerHTML = '<div class="media-empty">Searching YouTube Music…</div>';
|
||
try {
|
||
const data = await mediaPost("/search", { query, limit: 6 });
|
||
renderMediaResults(data.results);
|
||
} catch (e) {
|
||
mediaResultsEl.innerHTML = `<div class="media-empty">Search failed: ${escapeHtml(e.message)}</div>`;
|
||
}
|
||
});
|
||
|
||
function renderMediaChoices(choices) {
|
||
mediaChoicesWrap.hidden = choices.length === 0;
|
||
mediaChoicesEl.innerHTML = "";
|
||
for (const c of choices) {
|
||
const row = document.createElement("div");
|
||
row.className = "media-row media-choice";
|
||
const info = document.createElement("div");
|
||
info.className = "media-row-info";
|
||
const dur = fmtListDuration(c.duration);
|
||
info.innerHTML = `<strong>${escapeHtml(c.title)}</strong>` +
|
||
`<span>${escapeHtml(c.channel || "")}${dur ? " · " + dur : ""}</span>`;
|
||
const play = mediaRowActions();
|
||
play.addEventListener("click", () => playVideoId(c));
|
||
row.appendChild(info);
|
||
row.appendChild(play);
|
||
mediaChoicesEl.appendChild(row);
|
||
}
|
||
}
|
||
|
||
async function refreshLibrary() {
|
||
let data;
|
||
try {
|
||
data = await fetch("/media/api/library").then((r) => r.json());
|
||
} catch (e) {
|
||
return;
|
||
}
|
||
mediaLibraryEl.innerHTML = "";
|
||
const tracks = data.tracks || [];
|
||
if (tracks.length === 0) {
|
||
mediaLibraryEl.innerHTML = '<div class="media-empty">Downloaded songs and streams live here.</div>';
|
||
return;
|
||
}
|
||
for (const t of tracks) {
|
||
const row = document.createElement("div");
|
||
row.className = "media-row";
|
||
const info = document.createElement("div");
|
||
info.className = "media-row-info";
|
||
const dur = fmtListDuration(t.duration);
|
||
const statusTag = t.status === "ready" ? "" : ` · ${t.status}`;
|
||
info.innerHTML = `<strong>${escapeHtml(t.title)}</strong>` +
|
||
`<span>${escapeHtml(t.channel || (t.kind === "stream" ? "Live stream" : ""))}` +
|
||
`${dur ? " · " + dur : ""}${statusTag}</span>`;
|
||
const play = mediaRowActions();
|
||
play.disabled = t.status !== "ready";
|
||
play.addEventListener("click", () => {
|
||
mediaPost("/play", { track_id: t.id }).catch((e) => {
|
||
mediaChannelEl.textContent = `Could not play: ${e.message}`;
|
||
});
|
||
});
|
||
const del = document.createElement("button");
|
||
del.className = "btn btn-icon media-row-delete";
|
||
del.type = "button";
|
||
del.setAttribute("aria-label", `Remove ${t.title}`);
|
||
del.title = "Remove from library";
|
||
del.innerHTML = '<svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><path d="M18 6 6 18M6 6l12 12"></path></svg>';
|
||
del.addEventListener("click", async () => {
|
||
try {
|
||
await fetch(`/media/api/tracks/${encodeURIComponent(t.id)}`, { method: "DELETE" });
|
||
if (t.id === mediaCurrentTrackId) clearMediaAudio();
|
||
refreshLibrary();
|
||
} catch (e) { /* leave the row; next refresh decides */ }
|
||
});
|
||
row.appendChild(info);
|
||
row.appendChild(play);
|
||
row.appendChild(del);
|
||
mediaLibraryEl.appendChild(row);
|
||
}
|
||
}
|
||
|
||
streamForm.addEventListener("submit", async (event) => {
|
||
event.preventDefault();
|
||
const name = streamNameInput.value.trim() || streamUrlInput.value.trim();
|
||
const url = streamUrlInput.value.trim();
|
||
if (!url) return;
|
||
try {
|
||
await mediaPost("/streams", { name, url });
|
||
streamNameInput.value = "";
|
||
streamUrlInput.value = "";
|
||
refreshLibrary();
|
||
} catch (e) {
|
||
mediaChannelEl.textContent = `Could not add stream: ${e.message}`;
|
||
}
|
||
});
|
||
|
||
// ── Panel open/close ────────────────────────────────────────────────────────
|
||
function setMediaOpen(open) {
|
||
mediaPanel.classList.toggle("open", open);
|
||
mediaBtn.setAttribute("aria-expanded", String(open));
|
||
if (open) {
|
||
refreshLibrary();
|
||
pollMediaState();
|
||
}
|
||
}
|
||
|
||
mediaBtn.setAttribute("aria-expanded", "false");
|
||
mediaBtn.addEventListener("click", () => setMediaOpen(!mediaPanel.classList.contains("open")));
|
||
closeMediaBtn.addEventListener("click", () => setMediaOpen(false));
|
||
|
||
startMediaPolling();
|
||
updateMediaControls();
|
||
buildLedWall();
|