Shane
44e8d05e2c
- agent/task_registry.py: file-based JSONL event registry (cross-process) - agent/task_worker.py: autonomous LLM loop with weather/time/memory/web tools - agent/dispatch_mcp.py: MCP tool exposing dispatch_task to the main agent - agent/agent.py: registers dispatch toolset, polls task events → room data - web: slide-out task panel (FAB button + badge), live step streaming via data channel topic 'tasks', status dots (running/completed/failed) - Dockerfile: copies new task_*.py and dispatch_mcp.py files The dispatch MCP runs in its own process; events flow through /tmp/tasks/events.jsonl which the main agent tails every second and forwards to the browser. Tasks run up to 10 LLM iterations with tool calls.
751 lines
29 KiB
JavaScript
751 lines
29 KiB
JavaScript
// Voice Assistant — LiveKit client
|
|
const { Room, RoomEvent } = LivekitClient;
|
|
|
|
// ── Config ──────────────────────────────────────────────────────────────────
|
|
// LiveKit is reached through the same origin as the page (nginx proxies
|
|
// /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed
|
|
// content when the UI is served over HTTPS.
|
|
const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`;
|
|
const ROOM_NAME = "voice-room";
|
|
const AGENT_NAME = "voice-assistant";
|
|
|
|
const VOICES = [
|
|
{ id: "en-US-AvaNeural", name: "Ava", tag: "DragonHD" },
|
|
{ id: "en-US-JennyNeural", name: "Jenny" },
|
|
{ id: "en-US-GuyNeural", name: "Guy" },
|
|
{ id: "en-US-AndrewNeural", name: "Andrew" },
|
|
{ id: "en-US-AriaNeural", name: "Aria" },
|
|
{ id: "en-US-EmmaNeural", name: "Emma" },
|
|
{ id: "en-US-EricNeural", name: "Eric" },
|
|
{ id: "en-US-BrianNeural", name: "Brian" },
|
|
{ id: "en-US-AshleyNeural", name: "Ashley" },
|
|
{ id: "en-US-RichardNeural", name: "Richard" },
|
|
{ id: "en-US-TinaNeural", name: "Tina" },
|
|
{ id: "en-US-SteffanNeural", name: "Steffan" },
|
|
];
|
|
|
|
const HISTORY_KEY = "voice_history";
|
|
const HISTORY_MAX = 50;
|
|
|
|
// ── State ───────────────────────────────────────────────────────────────────
|
|
let room = null;
|
|
let agentParticipant = null;
|
|
let uiState = "idle"; // idle | connecting | listening | thinking | speaking
|
|
let micLevel = 0; // 0..1, smoothed
|
|
let agentAudioLevel = 0; // 0..1, smoothed
|
|
let userSpeaking = false; // local VAD-ish flag from mic level
|
|
let wakeLock = null;
|
|
|
|
function escapeHtml(s) {
|
|
const d = document.createElement("div");
|
|
d.textContent = s || "";
|
|
return d.innerHTML;
|
|
}
|
|
|
|
// ── DOM ─────────────────────────────────────────────────────────────────────
|
|
const startBtn = document.getElementById("startBtn");
|
|
const stopBtn = document.getElementById("stopBtn");
|
|
const muteBtn = document.getElementById("muteBtn");
|
|
let micMuted = false;
|
|
const statusEl = document.getElementById("status");
|
|
const messagesEl = document.getElementById("messages");
|
|
const orbEl = document.getElementById("orb");
|
|
const stateLabelEl = document.getElementById("stateLabel");
|
|
const thinkingChipEl = document.getElementById("thinkingChip");
|
|
const bargeHintEl = document.getElementById("bargeHint");
|
|
const micMeterFillEl = document.getElementById("micMeterFill");
|
|
const settingsBtn = document.getElementById("settingsBtn");
|
|
const settingsSheet = document.getElementById("settingsSheet");
|
|
const sheetBackdrop = document.getElementById("sheetBackdrop");
|
|
const closeSheetBtn = document.getElementById("closeSheetBtn");
|
|
const voiceListEl = document.getElementById("voiceList");
|
|
const clearBtn = document.getElementById("clearBtn");
|
|
const tasksBtn = document.getElementById("tasksBtn");
|
|
const tasksPanel = document.getElementById("tasksPanel");
|
|
const tasksList = document.getElementById("tasksList");
|
|
const tasksBadge = document.getElementById("tasksBadge");
|
|
const closeTasksBtn = document.getElementById("closeTasksBtn");
|
|
|
|
// ── Background tasks state ──────────────────────────────────────────────────
|
|
const tasks = new Map(); // task_id -> {id, description, status, steps: [], result, error}
|
|
|
|
function updateTasksBadge() {
|
|
const active = [...tasks.values()].filter(t => t.status === "running" || t.status === "pending").length;
|
|
if (active > 0) {
|
|
tasksBadge.hidden = false;
|
|
tasksBadge.textContent = String(active);
|
|
} else {
|
|
tasksBadge.hidden = true;
|
|
}
|
|
}
|
|
|
|
function renderTasks() {
|
|
const items = [...tasks.values()].sort((a, b) => (b.created_at || 0) - (a.created_at || 0));
|
|
if (items.length === 0) {
|
|
tasksList.innerHTML = '<p class="tasks-empty">No tasks yet. Ask Hope to research something.</p>';
|
|
return;
|
|
}
|
|
tasksList.innerHTML = "";
|
|
for (const t of items) {
|
|
const card = document.createElement("div");
|
|
card.className = "task-card";
|
|
const statusClass = t.status || "pending";
|
|
card.innerHTML = `
|
|
<div class="task-card-header">
|
|
<span class="task-status-dot ${statusClass}"></span>
|
|
<span class="task-desc">${escapeHtml(t.description)}</span>
|
|
</div>
|
|
<ul class="task-steps">${t.steps.map(s => `<li class="task-step ${s.role}">${escapeHtml(s.content)}</li>`).join("")}</ul>
|
|
${t.result ? `<div class="task-result">${escapeHtml(t.result)}</div>` : ""}
|
|
${t.error ? `<div class="task-error">${escapeHtml(t.error)}</div>` : ""}
|
|
`;
|
|
tasksList.appendChild(card);
|
|
}
|
|
}
|
|
|
|
function handleTaskEvent(data) {
|
|
const { task_id, event, role, content, result, error, task } = data;
|
|
if (event === "created" && task) {
|
|
tasks.set(task_id, { ...task, steps: [] });
|
|
} else if (event === "step") {
|
|
const t = tasks.get(task_id);
|
|
if (t) {
|
|
t.steps.push({ role, content });
|
|
t.status = "running";
|
|
}
|
|
} else if (event === "completed") {
|
|
const t = tasks.get(task_id);
|
|
if (t) { t.status = "completed"; t.result = result; }
|
|
} else if (event === "failed") {
|
|
const t = tasks.get(task_id);
|
|
if (t) { t.status = "failed"; t.error = error; }
|
|
}
|
|
updateTasksBadge();
|
|
renderTasks();
|
|
}
|
|
|
|
tasksBtn.addEventListener("click", () => tasksPanel.classList.toggle("open"));
|
|
closeTasksBtn.addEventListener("click", () => tasksPanel.classList.remove("open"));
|
|
|
|
// ── Audio analysers (mic + remote agent audio) ─────────────────────────────
|
|
let micCtx = null;
|
|
let micAnalyser = null;
|
|
let micBuf = null;
|
|
let remoteCtx = null;
|
|
let remoteAnalyser = null;
|
|
let remoteFreqBuf = null;
|
|
|
|
function setupMicAnalyser() {
|
|
try {
|
|
const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
|
|
const mst = pub && pub.track && pub.track.mediaStreamTrack;
|
|
if (mst) {
|
|
micCtx = new AudioContext();
|
|
micAnalyser = micCtx.createAnalyser();
|
|
micAnalyser.fftSize = 512;
|
|
micBuf = new Float32Array(micAnalyser.fftSize);
|
|
micCtx.createMediaStreamSource(new MediaStream([mst])).connect(micAnalyser);
|
|
}
|
|
} catch (e) { /* fall back to SDK audioLevel in the meter loop */ }
|
|
}
|
|
|
|
function setupRemoteAudioAnalyser(audioEl) {
|
|
try {
|
|
if (!audioEl.srcObject) return;
|
|
remoteCtx = new AudioContext();
|
|
remoteAnalyser = remoteCtx.createAnalyser();
|
|
remoteAnalyser.fftSize = 256;
|
|
remoteFreqBuf = new Uint8Array(remoteAnalyser.frequencyBinCount);
|
|
remoteCtx.createMediaStreamSource(audioEl.srcObject).connect(remoteAnalyser);
|
|
} catch (e) { /* analyser optional */ }
|
|
}
|
|
|
|
function teardownAudio() {
|
|
for (const ctx of [micCtx, remoteCtx]) {
|
|
if (ctx && ctx.state !== "closed") ctx.close().catch(() => {});
|
|
}
|
|
micCtx = micAnalyser = micBuf = null;
|
|
remoteCtx = remoteAnalyser = remoteFreqBuf = null;
|
|
}
|
|
|
|
function readMicLevel() {
|
|
let level = 0;
|
|
if (micAnalyser && micBuf) {
|
|
micAnalyser.getFloatTimeDomainData(micBuf);
|
|
for (let i = 0; i < micBuf.length; i++) level = Math.max(level, Math.abs(micBuf[i]));
|
|
} else if (room && room.localParticipant) {
|
|
level = room.localParticipant.audioLevel || 0;
|
|
}
|
|
// Smooth so the orb pulses rather than flickers
|
|
micLevel = micLevel * 0.6 + level * 0.4;
|
|
return micLevel;
|
|
}
|
|
|
|
function readAgentAudioLevel() {
|
|
let level = 0;
|
|
if (remoteAnalyser && remoteFreqBuf) {
|
|
remoteAnalyser.getByteFrequencyData(remoteFreqBuf);
|
|
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
|
level /= 255;
|
|
} else if (agentParticipant && agentParticipant.audioLevel !== undefined) {
|
|
level = agentParticipant.audioLevel || 0;
|
|
}
|
|
agentAudioLevel = agentAudioLevel * 0.6 + level * 0.4;
|
|
return agentAudioLevel;
|
|
}
|
|
|
|
// ── State machine ───────────────────────────────────────────────────────────
|
|
const STATE_LABELS = {
|
|
idle: "Idle",
|
|
connecting: "Connecting",
|
|
listening: "Listening",
|
|
thinking: "Thinking",
|
|
speaking: "Speaking",
|
|
};
|
|
|
|
function setState(next) {
|
|
if (next === uiState) return;
|
|
uiState = next;
|
|
orbEl.dataset.state = next;
|
|
stateLabelEl.textContent = STATE_LABELS[next] || next;
|
|
|
|
// Thinking timer: show the "thinking..." chip if it lingers past 2s
|
|
if (next === "thinking") {
|
|
thinkingChipEl.hidden = true;
|
|
setTimeout(() => {
|
|
if (uiState === "thinking") thinkingChipEl.hidden = false;
|
|
}, 2000);
|
|
} else {
|
|
thinkingChipEl.hidden = true;
|
|
}
|
|
|
|
// Barge-in hint only while the agent is speaking
|
|
bargeHintEl.hidden = next !== "speaking";
|
|
|
|
if (next === "idle") {
|
|
micLevel = 0;
|
|
agentAudioLevel = 0;
|
|
userSpeaking = false;
|
|
}
|
|
}
|
|
|
|
function inferState() {
|
|
if (!room || !room.localParticipant) return "idle";
|
|
if (!agentParticipant) return "connecting";
|
|
readMicLevel();
|
|
readAgentAudioLevel();
|
|
userSpeaking = micLevel > 0.035;
|
|
if (agentAudioLevel > 0.02) return "speaking";
|
|
if (userSpeaking) return "listening";
|
|
// Not speaking, agent silent: stay in thinking until audio arrives,
|
|
// otherwise settle back to listening.
|
|
if (uiState === "thinking" || uiState === "speaking") return "thinking";
|
|
return "listening";
|
|
}
|
|
|
|
// ── Visual loop: drives orb + meter + state inference ───────────────────────
|
|
let visualTimer = null;
|
|
function startVisualLoop() {
|
|
if (visualTimer) clearInterval(visualTimer);
|
|
visualTimer = setInterval(() => {
|
|
if (!room || !room.localParticipant) return;
|
|
setState(inferState());
|
|
|
|
// Mic meter (thin bar under the status line)
|
|
micMeterFillEl.style.width = `${Math.min(100, Math.round(micLevel * 250))}%`;
|
|
|
|
// Orb pulse: scale/opacity from whichever stream is active
|
|
if (uiState === "speaking") {
|
|
orbEl.style.setProperty("--pulse-scale", (1 + agentAudioLevel * 0.35).toFixed(3));
|
|
orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.55 + agentAudioLevel).toFixed(3));
|
|
} else if (uiState === "listening" && userSpeaking) {
|
|
orbEl.style.setProperty("--pulse-scale", (1 + micLevel * 0.4).toFixed(3));
|
|
orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.5 + micLevel).toFixed(3));
|
|
} else {
|
|
orbEl.style.setProperty("--pulse-scale", "1");
|
|
orbEl.style.setProperty("--pulse-opacity", "");
|
|
}
|
|
|
|
// Agent audio visualizer bars on the latest agent bubble
|
|
const viz = messagesEl.querySelector(".message-row.agent:last-of-type .audio-viz");
|
|
if (viz) {
|
|
let level = 0;
|
|
if (remoteAnalyser && remoteFreqBuf) {
|
|
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
|
level /= 255;
|
|
}
|
|
const bars = viz.children;
|
|
for (let i = 0; i < bars.length; i++) {
|
|
// Per-bar phase offset so the bars dance rather than move in lockstep
|
|
const h = Math.max(0.15, level * (0.6 + 0.4 * Math.sin(Date.now() / 120 + i * 1.3)));
|
|
bars[i].style.height = `${h * 100}%`;
|
|
}
|
|
}
|
|
}, 80);
|
|
}
|
|
|
|
function stopVisualLoop() {
|
|
if (visualTimer) clearInterval(visualTimer);
|
|
visualTimer = null;
|
|
micMeterFillEl.style.width = "0%";
|
|
orbEl.style.setProperty("--pulse-scale", "1");
|
|
orbEl.style.setProperty("--pulse-opacity", "");
|
|
}
|
|
|
|
// ── Conversation history (localStorage) ─────────────────────────────────────
|
|
function loadHistory() {
|
|
try {
|
|
const raw = localStorage.getItem(HISTORY_KEY);
|
|
if (!raw) return [];
|
|
const arr = JSON.parse(raw);
|
|
return Array.isArray(arr) ? arr.slice(-HISTORY_MAX) : [];
|
|
} catch (e) { return []; }
|
|
}
|
|
|
|
function persistHistory() {
|
|
try {
|
|
localStorage.setItem(HISTORY_KEY, JSON.stringify(history.slice(-HISTORY_MAX)));
|
|
} catch (e) { /* storage full or unavailable */ }
|
|
}
|
|
|
|
let history = loadHistory();
|
|
|
|
function renderMessage(msg) {
|
|
const wrap = document.createElement("div");
|
|
wrap.className = `message-row ${msg.role}`;
|
|
wrap.dataset.ts = msg.ts || "";
|
|
|
|
if (msg.role === "agent") {
|
|
const dot = document.createElement("span");
|
|
dot.className = "avatar-dot";
|
|
wrap.appendChild(dot);
|
|
}
|
|
|
|
const bubble = document.createElement("div");
|
|
bubble.className = "bubble";
|
|
bubble.textContent = msg.text || "";
|
|
if (msg.role === "agent") {
|
|
const viz = document.createElement("span");
|
|
viz.className = "audio-viz";
|
|
for (let i = 0; i < 5; i++) viz.appendChild(document.createElement("i"));
|
|
bubble.appendChild(viz);
|
|
}
|
|
wrap.appendChild(bubble);
|
|
messagesEl.appendChild(wrap);
|
|
messagesEl.scrollTop = messagesEl.scrollHeight;
|
|
}
|
|
|
|
function addMessage(role, text) {
|
|
clearPartials();
|
|
const msg = { role, text, ts: Date.now() };
|
|
history.push(msg);
|
|
persistHistory();
|
|
renderMessage(msg);
|
|
return msg;
|
|
}
|
|
|
|
// Partial transcripts render greyed and update in place until a final arrives.
|
|
function addPartial(role, text) {
|
|
const rows = messagesEl.querySelectorAll(".message-row.partial");
|
|
let last = null;
|
|
for (const r of rows) last = r;
|
|
if (last && last.dataset.role === role) {
|
|
last.querySelector(".bubble").textContent = text;
|
|
messagesEl.scrollTop = messagesEl.scrollHeight;
|
|
return;
|
|
}
|
|
const wrap = document.createElement("div");
|
|
wrap.className = `message-row ${role} partial`;
|
|
wrap.dataset.role = role;
|
|
if (role === "agent") {
|
|
const dot = document.createElement("span");
|
|
dot.className = "avatar-dot";
|
|
wrap.appendChild(dot);
|
|
}
|
|
const bubble = document.createElement("div");
|
|
bubble.className = "bubble partial";
|
|
bubble.textContent = text;
|
|
wrap.appendChild(bubble);
|
|
messagesEl.appendChild(wrap);
|
|
messagesEl.scrollTop = messagesEl.scrollHeight;
|
|
}
|
|
|
|
function clearPartials() {
|
|
messagesEl.querySelectorAll(".message-row.partial").forEach((el) => el.remove());
|
|
}
|
|
|
|
function renderHistory() {
|
|
messagesEl.innerHTML = "";
|
|
for (const msg of history) renderMessage(msg);
|
|
messagesEl.scrollTop = messagesEl.scrollHeight;
|
|
}
|
|
|
|
clearBtn.addEventListener("click", () => {
|
|
history = [];
|
|
persistHistory();
|
|
clearPartials();
|
|
renderHistory();
|
|
});
|
|
|
|
renderHistory();
|
|
|
|
// ── Transcripts ─────────────────────────────────────────────────────────────
|
|
// Primary: livekit-agents built-in text stream (word-by-word when available).
|
|
function registerTranscriptionStream(rm) {
|
|
try {
|
|
rm.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => {
|
|
const text = await reader.readAll();
|
|
if (!text || !text.trim()) return;
|
|
const attrs = (reader.info && reader.info.attributes) || {};
|
|
const trackId = attrs["lk.transcribed_track_id"] || "";
|
|
// Distinguish user vs agent by the transcribed track's owner.
|
|
let role = "agent";
|
|
if (participantInfo && participantInfo.identity === AGENT_NAME) {
|
|
role = "agent";
|
|
} else if (trackId) {
|
|
for (const p of rm.remoteParticipants.values()) {
|
|
const pub = p.getTrackPublication(trackId);
|
|
if (pub) { role = isAgent(p) ? "agent" : "user"; break; }
|
|
}
|
|
}
|
|
// Use partial rendering: updates the same row in place until final.
|
|
addPartial(role, text.trim());
|
|
});
|
|
} catch (e) {
|
|
// Older SDKs may not support registerTextStreamHandler — the data
|
|
// channel handler below still covers transcripts.
|
|
console.warn("text stream registration failed:", e);
|
|
}
|
|
}
|
|
|
|
// Secondary: agent publishes {type:"transcript"} on topic "transcript".
|
|
function handleDataPacket(payload, participant, topic) {
|
|
try {
|
|
const msg = JSON.parse(new TextDecoder().decode(payload));
|
|
if (msg.type === "transcript") {
|
|
addMessage(msg.role || "agent", msg.text);
|
|
} else if (msg.type === "task_event") {
|
|
handleTaskEvent(msg);
|
|
}
|
|
// set_voice messages flow the other way; nothing to do client-side.
|
|
} catch (e) {
|
|
console.warn("bad data packet", e);
|
|
}
|
|
}
|
|
|
|
// ── Voice settings sheet ────────────────────────────────────────────────────
|
|
function currentVoiceId() {
|
|
const saved = localStorage.getItem("preferred_voice");
|
|
return VOICES.some((v) => v.id === saved) ? saved : VOICES[0].id;
|
|
}
|
|
|
|
function renderVoiceList() {
|
|
voiceListEl.innerHTML = "";
|
|
const active = currentVoiceId();
|
|
for (const v of VOICES) {
|
|
const card = document.createElement("div");
|
|
card.className = "voice-card" + (v.id === active ? " active" : "");
|
|
card.dataset.voice = v.id;
|
|
|
|
const name = document.createElement("span");
|
|
name.className = "voice-name";
|
|
name.textContent = v.name + (v.tag ? ` (${v.tag})` : "");
|
|
|
|
const previewBtn = document.createElement("button");
|
|
previewBtn.className = "btn btn-ghost voice-preview";
|
|
previewBtn.setAttribute("aria-label", `Preview ${v.name}`);
|
|
previewBtn.title = "Preview (coming soon)";
|
|
// TODO: play a short Azure TTS sample for this voice.
|
|
previewBtn.addEventListener("click", (e) => {
|
|
e.stopPropagation();
|
|
addMessage("system", `Preview for ${v.name} is not wired up yet.`);
|
|
});
|
|
|
|
card.appendChild(name);
|
|
card.appendChild(previewBtn);
|
|
card.addEventListener("click", () => selectVoice(v.id));
|
|
voiceListEl.appendChild(card);
|
|
}
|
|
}
|
|
|
|
function selectVoice(voiceId) {
|
|
localStorage.setItem("preferred_voice", voiceId);
|
|
renderVoiceList();
|
|
if (room && agentParticipant) {
|
|
sendVoiceMessage(voiceId);
|
|
const v = VOICES.find((x) => x.id === voiceId);
|
|
addMessage("system", `Voice changed to ${v ? v.name : voiceId}`);
|
|
}
|
|
}
|
|
|
|
function openSheet() {
|
|
renderVoiceList();
|
|
sheetBackdrop.hidden = false;
|
|
settingsSheet.classList.add("open");
|
|
}
|
|
|
|
function closeSheet() {
|
|
settingsSheet.classList.remove("open");
|
|
sheetBackdrop.hidden = true;
|
|
}
|
|
|
|
settingsBtn.addEventListener("click", openSheet);
|
|
closeSheetBtn.addEventListener("click", closeSheet);
|
|
sheetBackdrop.addEventListener("click", closeSheet);
|
|
document.addEventListener("keydown", (e) => {
|
|
if (e.key === "Escape") closeSheet();
|
|
});
|
|
|
|
// ── Wake lock ───────────────────────────────────────────────────────────────
|
|
async function acquireWakeLock() {
|
|
try {
|
|
if ("wakeLock" in navigator) {
|
|
wakeLock = await navigator.wakeLock.request("screen");
|
|
}
|
|
} catch (e) { /* not critical */ }
|
|
}
|
|
|
|
function releaseWakeLock() {
|
|
if (wakeLock) {
|
|
wakeLock.release().catch(() => {});
|
|
wakeLock = null;
|
|
}
|
|
}
|
|
|
|
// ── Agent detection ─────────────────────────────────────────────────────────
|
|
// The agent may already be in the room before we join (it persists between
|
|
// page loads), so scan remoteParticipants after connect AND watch for new
|
|
// joins — ParticipantConnected does not fire for pre-existing participants.
|
|
function isAgent(participant) {
|
|
// kind is the numeric proto enum; ParticipantKind.AGENT === 4.
|
|
// Prefer the SDK's own getter where available.
|
|
if (participant.isAgent !== undefined && participant.isAgent !== null) {
|
|
return participant.isAgent;
|
|
}
|
|
return participant.identity === AGENT_NAME
|
|
|| participant.kind === LivekitClient.ParticipantKind.AGENT;
|
|
}
|
|
|
|
function findAgent(rm) {
|
|
if (!rm) return null;
|
|
for (const p of rm.remoteParticipants.values()) {
|
|
if (isAgent(p)) return p;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function markAgentFound(rm, participant) {
|
|
agentParticipant = participant;
|
|
statusEl.textContent = "Agent connected — speak now";
|
|
// Send current voice selection to the agent
|
|
sendVoiceMessage(currentVoiceId(), rm);
|
|
}
|
|
|
|
function createRoom(token) {
|
|
const newRoom = new Room({
|
|
adaptiveStream: true,
|
|
dynacast: true,
|
|
});
|
|
|
|
// Listen for the agent joining after us
|
|
newRoom.on(RoomEvent.ParticipantConnected, (participant) => {
|
|
if (isAgent(participant)) markAgentFound(newRoom, participant);
|
|
});
|
|
|
|
// Attach and play incoming audio tracks (agent TTS)
|
|
newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
|
|
if (track.kind === LivekitClient.Track.Kind.Audio) {
|
|
const el = track.attach();
|
|
el.id = `audio-${participant.identity}`;
|
|
document.body.appendChild(el);
|
|
// Remote audio analyser drives the orb + visualizer while speaking
|
|
setupRemoteAudioAnalyser(el);
|
|
}
|
|
});
|
|
|
|
newRoom.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
track.detach().forEach((el) => el.remove());
|
|
});
|
|
|
|
// Browsers block audio playback until a user gesture; show an unlock
|
|
// button if the page loads without one.
|
|
newRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => {
|
|
if (!newRoom.canPlaybackAudio) {
|
|
const existing = document.getElementById("audioUnlockBtn");
|
|
if (!existing) {
|
|
const btn = document.createElement("button");
|
|
btn.id = "audioUnlockBtn";
|
|
btn.className = "btn btn-primary";
|
|
btn.textContent = "Tap to enable audio";
|
|
btn.style.cssText = "position:fixed;bottom:2rem;left:50%;transform:translateX(-50%);z-index:999;padding:1rem 2rem;font-size:1.1rem;border-radius:999px;box-shadow:0 4px 24px rgba(0,0,0,.5)";
|
|
btn.onclick = () => { newRoom.startAudio(); btn.remove(); };
|
|
document.body.appendChild(btn);
|
|
}
|
|
} else {
|
|
const existing = document.getElementById("audioUnlockBtn");
|
|
if (existing) existing.remove();
|
|
}
|
|
});
|
|
|
|
// Live text stream (word-by-word transcripts)
|
|
registerTranscriptionStream(newRoom);
|
|
|
|
// Data channel: transcripts (secondary source) + voice control
|
|
newRoom.on(RoomEvent.DataReceived, (payload, participant, kind, topic) => {
|
|
handleDataPacket(payload, participant, topic);
|
|
});
|
|
|
|
// Guard with `=== room` so a stale (retry-discarded) connection cannot
|
|
// clobber UI state of the active one.
|
|
newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
|
|
if (participant === newRoom.localParticipant && newRoom === room) {
|
|
statusEl.textContent = `Connected (${quality} quality)`;
|
|
}
|
|
});
|
|
|
|
newRoom.on(RoomEvent.Disconnected, () => {
|
|
if (newRoom !== room) return;
|
|
teardownAudio();
|
|
stopVisualLoop();
|
|
releaseWakeLock();
|
|
statusEl.textContent = "Disconnected";
|
|
setState("idle");
|
|
startBtn.disabled = false;
|
|
stopBtn.disabled = true;
|
|
agentParticipant = null;
|
|
});
|
|
|
|
return newRoom.connect(LIVEKIT_URL, token, {
|
|
autoSubscribe: true,
|
|
}).then(() => {
|
|
// Pick up an agent that was already in the room before we joined
|
|
const existing = findAgent(newRoom);
|
|
if (existing) markAgentFound(newRoom, existing);
|
|
return newRoom;
|
|
});
|
|
}
|
|
|
|
function waitForAgent(timeoutMs) {
|
|
return new Promise((resolve) => {
|
|
if (findAgent(room)) return resolve(true);
|
|
const deadline = Date.now() + timeoutMs;
|
|
const timer = setInterval(() => {
|
|
const found = findAgent(room);
|
|
if (found) {
|
|
if (!agentParticipant) markAgentFound(room, found);
|
|
clearInterval(timer);
|
|
resolve(true);
|
|
} else if (Date.now() > deadline) {
|
|
clearInterval(timer);
|
|
resolve(false);
|
|
}
|
|
}, 250);
|
|
});
|
|
}
|
|
|
|
startBtn.addEventListener("click", async () => {
|
|
try {
|
|
setState("connecting");
|
|
statusEl.textContent = "Connecting...";
|
|
await acquireWakeLock();
|
|
|
|
// Get a signed access token from the token endpoint
|
|
const token = await fetchToken(ROOM_NAME);
|
|
|
|
room = await createRoom(token);
|
|
await room.startAudio();
|
|
|
|
// The agent is dispatched when we join. If it misses the dispatch
|
|
// (e.g. the server just started and the worker wasn't ready yet),
|
|
// reconnect once to trigger a fresh dispatch.
|
|
statusEl.textContent = "Waiting for assistant...";
|
|
if (!(await waitForAgent(10000))) {
|
|
statusEl.textContent = "Assistant not ready, retrying...";
|
|
const stale = room;
|
|
room = null; // detach first so its Disconnected handler
|
|
agentParticipant = null; // cannot clobber the UI
|
|
await stale.disconnect();
|
|
const freshToken = await fetchToken(ROOM_NAME);
|
|
room = await createRoom(freshToken);
|
|
if (!(await waitForAgent(15000))) {
|
|
throw new Error("Assistant did not join — reload the page and try again");
|
|
}
|
|
}
|
|
|
|
statusEl.textContent = "Requesting microphone...";
|
|
await room.localParticipant.setMicrophoneEnabled(true);
|
|
|
|
statusEl.textContent = "Connected — speak now";
|
|
startBtn.disabled = true;
|
|
stopBtn.disabled = false;
|
|
muteBtn.disabled = false;
|
|
micMuted = false;
|
|
|
|
setupMicAnalyser();
|
|
startVisualLoop();
|
|
|
|
} catch (err) {
|
|
console.error("Connection failed:", err);
|
|
statusEl.textContent = `Error: ${err.message}`;
|
|
setState("idle");
|
|
releaseWakeLock();
|
|
}
|
|
});
|
|
|
|
// ── Stop ────────────────────────────────────────────────────────────────────
|
|
stopBtn.addEventListener("click", () => {
|
|
if (room) {
|
|
room.disconnect();
|
|
room = null;
|
|
agentParticipant = null;
|
|
}
|
|
teardownAudio();
|
|
stopVisualLoop();
|
|
releaseWakeLock();
|
|
document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove());
|
|
statusEl.textContent = "Disconnected";
|
|
setState("idle");
|
|
startBtn.disabled = false;
|
|
stopBtn.disabled = true;
|
|
muteBtn.disabled = true;
|
|
micMuted = false;
|
|
updateMuteIcon();
|
|
});
|
|
|
|
// ── Mic mute toggle ─────────────────────────────────────────────────────────
|
|
muteBtn.addEventListener("click", async () => {
|
|
if (!room || !room.localParticipant) return;
|
|
micMuted = !micMuted;
|
|
await room.localParticipant.setMicrophoneEnabled(!micMuted);
|
|
updateMuteIcon();
|
|
});
|
|
|
|
function updateMuteIcon() {
|
|
const muted = muteBtn.classList.toggle("muted", micMuted);
|
|
muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone");
|
|
muteBtn.title = muted ? "Unmute microphone" : "Mute microphone";
|
|
}
|
|
|
|
// ── Helper: fetch a signed access token from the token endpoint ────────────
|
|
async function fetchToken(roomName) {
|
|
const resp = await fetch("/token", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ room: roomName }),
|
|
});
|
|
if (!resp.ok) throw new Error(`Token endpoint returned ${resp.status}`);
|
|
const data = await resp.json();
|
|
return data.token;
|
|
}
|
|
|
|
// ── Helper: send voice selection to the agent via data channel ─────────────
|
|
function sendVoiceMessage(voice, targetRoom) {
|
|
const r = targetRoom || room;
|
|
if (!r || !r.localParticipant) return;
|
|
const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice }));
|
|
r.localParticipant.publishData(payload, {
|
|
topic: "voice-control",
|
|
reliable: true,
|
|
});
|
|
}
|