feat: implement full UPDATE.md review — critical fixes, UI upgrade, infra hardening
Critical frontend bugs: - Add TrackSubscribed/attach() for agent audio playback - Fix decodeToString TypeError with TextDecoder - XSS fix: innerHTML -> textContent in addMessage - Fresh token on reconnect retry Agent fixes: - GemmaLLM subclass with reasoning_content fallback wrapper - Disable Gemma 4 thinking mode via chat_template_kwargs (6.8s -> 0.5s) - Remove duplicate session-level LLM - Replace global _active_session with closure-based handler - asyncio.create_task instead of deprecated get_event_loop - Explicit silero VAD, topic filter on voice-control Infra: - supervisord: all programs log to /dev/stdout - Dockerfile: uv sync --frozen with committed uv.lock - nginx config moved to real file, token_server.py no longer served - entrypoint.sh: cert persisted, only regenerated on IP change - compose: healthcheck + cert volume - token_server: CORS removed, room pinned to voice-room UI upgrade: - Orb UI with state machine (idle/connecting/listening/thinking/speaking) - Streaming transcripts via lk.transcription text streams - Barge-in hint, thinking chip, audio visualizer - Glassmorphism, chat bubbles, settings sheet, light mode - PWA manifest, favicon, wake-lock, safe-area insets - localStorage conversation history Docs: AGENTS.md drift fixed
This commit is contained in:
Shane
+469
-70
@@ -9,32 +9,429 @@ const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://$
|
||||
const ROOM_NAME = "voice-room";
|
||||
const AGENT_NAME = "voice-assistant";
|
||||
|
||||
const VOICES = [
|
||||
{ id: "en-US-AvaNeural", name: "Ava", tag: "DragonHD" },
|
||||
{ id: "en-US-JennyNeural", name: "Jenny" },
|
||||
{ id: "en-US-GuyNeural", name: "Guy" },
|
||||
{ id: "en-US-AndrewNeural", name: "Andrew" },
|
||||
{ id: "en-US-AriaNeural", name: "Aria" },
|
||||
{ id: "en-US-EmmaNeural", name: "Emma" },
|
||||
{ id: "en-US-EricNeural", name: "Eric" },
|
||||
{ id: "en-US-BrianNeural", name: "Brian" },
|
||||
{ id: "en-US-AshleyNeural", name: "Ashley" },
|
||||
{ id: "en-US-RichardNeural", name: "Richard" },
|
||||
{ id: "en-US-TinaNeural", name: "Tina" },
|
||||
{ id: "en-US-SteffanNeural", name: "Steffan" },
|
||||
];
|
||||
|
||||
const HISTORY_KEY = "voice_history";
|
||||
const HISTORY_MAX = 50;
|
||||
|
||||
// ── State ───────────────────────────────────────────────────────────────────
|
||||
let room = null;
|
||||
let agentParticipant = null;
|
||||
let uiState = "idle"; // idle | connecting | listening | thinking | speaking
|
||||
let micLevel = 0; // 0..1, smoothed
|
||||
let agentAudioLevel = 0; // 0..1, smoothed
|
||||
let userSpeaking = false; // local VAD-ish flag from mic level
|
||||
let wakeLock = null;
|
||||
|
||||
// ── DOM ─────────────────────────────────────────────────────────────────────
|
||||
const startBtn = document.getElementById("startBtn");
|
||||
const stopBtn = document.getElementById("stopBtn");
|
||||
const statusEl = document.getElementById("status");
|
||||
const messagesEl = document.getElementById("messages");
|
||||
const voiceSelect = document.getElementById("voiceSelect");
|
||||
const orbEl = document.getElementById("orb");
|
||||
const stateLabelEl = document.getElementById("stateLabel");
|
||||
const thinkingChipEl = document.getElementById("thinkingChip");
|
||||
const bargeHintEl = document.getElementById("bargeHint");
|
||||
const micMeterFillEl = document.getElementById("micMeterFill");
|
||||
const settingsBtn = document.getElementById("settingsBtn");
|
||||
const settingsSheet = document.getElementById("settingsSheet");
|
||||
const sheetBackdrop = document.getElementById("sheetBackdrop");
|
||||
const closeSheetBtn = document.getElementById("closeSheetBtn");
|
||||
const voiceListEl = document.getElementById("voiceList");
|
||||
const clearBtn = document.getElementById("clearBtn");
|
||||
|
||||
// ── Voice switching ─────────────────────────────────────────────────────────
|
||||
voiceSelect.addEventListener("change", () => {
|
||||
const voice = voiceSelect.value;
|
||||
if (room && agentParticipant) {
|
||||
sendVoiceMessage(voice);
|
||||
addMessage("system", `Voice changed to ${voice}`);
|
||||
} else {
|
||||
localStorage.setItem("preferred_voice", voice);
|
||||
// ── Audio analysers (mic + remote agent audio) ─────────────────────────────
|
||||
let micCtx = null;
|
||||
let micAnalyser = null;
|
||||
let micBuf = null;
|
||||
let remoteCtx = null;
|
||||
let remoteAnalyser = null;
|
||||
let remoteFreqBuf = null;
|
||||
|
||||
function setupMicAnalyser() {
|
||||
try {
|
||||
const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
|
||||
const mst = pub && pub.track && pub.track.mediaStreamTrack;
|
||||
if (mst) {
|
||||
micCtx = new AudioContext();
|
||||
micAnalyser = micCtx.createAnalyser();
|
||||
micAnalyser.fftSize = 512;
|
||||
micBuf = new Float32Array(micAnalyser.fftSize);
|
||||
micCtx.createMediaStreamSource(new MediaStream([mst])).connect(micAnalyser);
|
||||
}
|
||||
} catch (e) { /* fall back to SDK audioLevel in the meter loop */ }
|
||||
}
|
||||
|
||||
function setupRemoteAudioAnalyser(audioEl) {
|
||||
try {
|
||||
if (!audioEl.srcObject) return;
|
||||
remoteCtx = new AudioContext();
|
||||
remoteAnalyser = remoteCtx.createAnalyser();
|
||||
remoteAnalyser.fftSize = 256;
|
||||
remoteFreqBuf = new Uint8Array(remoteAnalyser.frequencyBinCount);
|
||||
remoteCtx.createMediaStreamSource(audioEl.srcObject).connect(remoteAnalyser);
|
||||
} catch (e) { /* analyser optional */ }
|
||||
}
|
||||
|
||||
function teardownAudio() {
|
||||
for (const ctx of [micCtx, remoteCtx]) {
|
||||
if (ctx && ctx.state !== "closed") ctx.close().catch(() => {});
|
||||
}
|
||||
micCtx = micAnalyser = micBuf = null;
|
||||
remoteCtx = remoteAnalyser = remoteFreqBuf = null;
|
||||
}
|
||||
|
||||
function readMicLevel() {
|
||||
let level = 0;
|
||||
if (micAnalyser && micBuf) {
|
||||
micAnalyser.getFloatTimeDomainData(micBuf);
|
||||
for (let i = 0; i < micBuf.length; i++) level = Math.max(level, Math.abs(micBuf[i]));
|
||||
} else if (room && room.localParticipant) {
|
||||
level = room.localParticipant.audioLevel || 0;
|
||||
}
|
||||
// Smooth so the orb pulses rather than flickers
|
||||
micLevel = micLevel * 0.6 + level * 0.4;
|
||||
return micLevel;
|
||||
}
|
||||
|
||||
function readAgentAudioLevel() {
|
||||
let level = 0;
|
||||
if (remoteAnalyser && remoteFreqBuf) {
|
||||
remoteAnalyser.getByteFrequencyData(remoteFreqBuf);
|
||||
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
||||
level /= 255;
|
||||
} else if (agentParticipant && agentParticipant.audioLevel !== undefined) {
|
||||
level = agentParticipant.audioLevel || 0;
|
||||
}
|
||||
agentAudioLevel = agentAudioLevel * 0.6 + level * 0.4;
|
||||
return agentAudioLevel;
|
||||
}
|
||||
|
||||
// ── State machine ───────────────────────────────────────────────────────────
|
||||
const STATE_LABELS = {
|
||||
idle: "Idle",
|
||||
connecting: "Connecting",
|
||||
listening: "Listening",
|
||||
thinking: "Thinking",
|
||||
speaking: "Speaking",
|
||||
};
|
||||
|
||||
function setState(next) {
|
||||
if (next === uiState) return;
|
||||
uiState = next;
|
||||
orbEl.dataset.state = next;
|
||||
stateLabelEl.textContent = STATE_LABELS[next] || next;
|
||||
|
||||
// Thinking timer: show the "thinking..." chip if it lingers past 2s
|
||||
if (next === "thinking") {
|
||||
thinkingChipEl.hidden = true;
|
||||
setTimeout(() => {
|
||||
if (uiState === "thinking") thinkingChipEl.hidden = false;
|
||||
}, 2000);
|
||||
} else {
|
||||
thinkingChipEl.hidden = true;
|
||||
}
|
||||
|
||||
// Barge-in hint only while the agent is speaking
|
||||
bargeHintEl.hidden = next !== "speaking";
|
||||
|
||||
if (next === "idle") {
|
||||
micLevel = 0;
|
||||
agentAudioLevel = 0;
|
||||
userSpeaking = false;
|
||||
}
|
||||
}
|
||||
|
||||
function inferState() {
|
||||
if (!room || !room.localParticipant) return "idle";
|
||||
if (!agentParticipant) return "connecting";
|
||||
readMicLevel();
|
||||
readAgentAudioLevel();
|
||||
userSpeaking = micLevel > 0.035;
|
||||
if (agentAudioLevel > 0.02) return "speaking";
|
||||
if (userSpeaking) return "listening";
|
||||
// Not speaking, agent silent: stay in thinking until audio arrives,
|
||||
// otherwise settle back to listening.
|
||||
if (uiState === "thinking" || uiState === "speaking") return "thinking";
|
||||
return "listening";
|
||||
}
|
||||
|
||||
// ── Visual loop: drives orb + meter + state inference ───────────────────────
|
||||
let visualTimer = null;
|
||||
function startVisualLoop() {
|
||||
if (visualTimer) clearInterval(visualTimer);
|
||||
visualTimer = setInterval(() => {
|
||||
if (!room || !room.localParticipant) return;
|
||||
setState(inferState());
|
||||
|
||||
// Mic meter (thin bar under the status line)
|
||||
micMeterFillEl.style.width = `${Math.min(100, Math.round(micLevel * 250))}%`;
|
||||
|
||||
// Orb pulse: scale/opacity from whichever stream is active
|
||||
if (uiState === "speaking") {
|
||||
orbEl.style.setProperty("--pulse-scale", (1 + agentAudioLevel * 0.35).toFixed(3));
|
||||
orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.55 + agentAudioLevel).toFixed(3));
|
||||
} else if (uiState === "listening" && userSpeaking) {
|
||||
orbEl.style.setProperty("--pulse-scale", (1 + micLevel * 0.4).toFixed(3));
|
||||
orbEl.style.setProperty("--pulse-opacity", Math.min(1, 0.5 + micLevel).toFixed(3));
|
||||
} else {
|
||||
orbEl.style.setProperty("--pulse-scale", "1");
|
||||
orbEl.style.setProperty("--pulse-opacity", "");
|
||||
}
|
||||
|
||||
// Agent audio visualizer bars on the latest agent bubble
|
||||
const viz = messagesEl.querySelector(".message-row.agent:last-of-type .audio-viz");
|
||||
if (viz) {
|
||||
let level = 0;
|
||||
if (remoteAnalyser && remoteFreqBuf) {
|
||||
for (let i = 0; i < remoteFreqBuf.length; i++) level = Math.max(level, remoteFreqBuf[i]);
|
||||
level /= 255;
|
||||
}
|
||||
const bars = viz.children;
|
||||
for (let i = 0; i < bars.length; i++) {
|
||||
// Per-bar phase offset so the bars dance rather than move in lockstep
|
||||
const h = Math.max(0.15, level * (0.6 + 0.4 * Math.sin(Date.now() / 120 + i * 1.3)));
|
||||
bars[i].style.height = `${h * 100}%`;
|
||||
}
|
||||
}
|
||||
}, 80);
|
||||
}
|
||||
|
||||
function stopVisualLoop() {
|
||||
if (visualTimer) clearInterval(visualTimer);
|
||||
visualTimer = null;
|
||||
micMeterFillEl.style.width = "0%";
|
||||
orbEl.style.setProperty("--pulse-scale", "1");
|
||||
orbEl.style.setProperty("--pulse-opacity", "");
|
||||
}
|
||||
|
||||
// ── Conversation history (localStorage) ─────────────────────────────────────
|
||||
function loadHistory() {
|
||||
try {
|
||||
const raw = localStorage.getItem(HISTORY_KEY);
|
||||
if (!raw) return [];
|
||||
const arr = JSON.parse(raw);
|
||||
return Array.isArray(arr) ? arr.slice(-HISTORY_MAX) : [];
|
||||
} catch (e) { return []; }
|
||||
}
|
||||
|
||||
function persistHistory() {
|
||||
try {
|
||||
localStorage.setItem(HISTORY_KEY, JSON.stringify(history.slice(-HISTORY_MAX)));
|
||||
} catch (e) { /* storage full or unavailable */ }
|
||||
}
|
||||
|
||||
let history = loadHistory();
|
||||
|
||||
function renderMessage(msg) {
|
||||
const wrap = document.createElement("div");
|
||||
wrap.className = `message-row ${msg.role}`;
|
||||
wrap.dataset.ts = msg.ts || "";
|
||||
|
||||
if (msg.role === "agent") {
|
||||
const dot = document.createElement("span");
|
||||
dot.className = "avatar-dot";
|
||||
wrap.appendChild(dot);
|
||||
}
|
||||
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "bubble";
|
||||
bubble.textContent = msg.text || "";
|
||||
if (msg.role === "agent") {
|
||||
const viz = document.createElement("span");
|
||||
viz.className = "audio-viz";
|
||||
for (let i = 0; i < 5; i++) viz.appendChild(document.createElement("i"));
|
||||
bubble.appendChild(viz);
|
||||
}
|
||||
wrap.appendChild(bubble);
|
||||
messagesEl.appendChild(wrap);
|
||||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||||
}
|
||||
|
||||
function addMessage(role, text) {
|
||||
const msg = { role, text, ts: Date.now() };
|
||||
history.push(msg);
|
||||
persistHistory();
|
||||
renderMessage(msg);
|
||||
return msg;
|
||||
}
|
||||
|
||||
// Partial transcripts render greyed and update in place until a final arrives.
|
||||
function addPartial(role, text) {
|
||||
const rows = messagesEl.querySelectorAll(".message-row.partial");
|
||||
let last = null;
|
||||
for (const r of rows) last = r;
|
||||
if (last && last.dataset.role === role) {
|
||||
last.querySelector(".bubble").textContent = text;
|
||||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||||
return;
|
||||
}
|
||||
const wrap = document.createElement("div");
|
||||
wrap.className = `message-row ${role} partial`;
|
||||
wrap.dataset.role = role;
|
||||
if (role === "agent") {
|
||||
const dot = document.createElement("span");
|
||||
dot.className = "avatar-dot";
|
||||
wrap.appendChild(dot);
|
||||
}
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "bubble partial";
|
||||
bubble.textContent = text;
|
||||
wrap.appendChild(bubble);
|
||||
messagesEl.appendChild(wrap);
|
||||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||||
}
|
||||
|
||||
function clearPartials() {
|
||||
messagesEl.querySelectorAll(".message-row.partial").forEach((el) => el.remove());
|
||||
}
|
||||
|
||||
function renderHistory() {
|
||||
messagesEl.innerHTML = "";
|
||||
for (const msg of history) renderMessage(msg);
|
||||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||||
}
|
||||
|
||||
clearBtn.addEventListener("click", () => {
|
||||
history = [];
|
||||
persistHistory();
|
||||
clearPartials();
|
||||
renderHistory();
|
||||
});
|
||||
|
||||
// Restore saved voice preference
|
||||
const savedVoice = localStorage.getItem("preferred_voice");
|
||||
if (savedVoice) {
|
||||
voiceSelect.value = savedVoice;
|
||||
renderHistory();
|
||||
|
||||
// ── Transcripts ─────────────────────────────────────────────────────────────
|
||||
// Primary: livekit-agents built-in text stream (word-by-word when available).
|
||||
function registerTranscriptionStream(rm) {
|
||||
try {
|
||||
rm.registerTextStreamHandler("lk.transcription", async (reader, participantInfo) => {
|
||||
const text = await reader.readAll();
|
||||
if (!text || !text.trim()) return;
|
||||
const attrs = (reader.info && reader.info.attributes) || {};
|
||||
const trackId = attrs["lk.transcribed_track_id"] || "";
|
||||
// Distinguish user vs agent by the transcribed track's owner.
|
||||
let role = "agent";
|
||||
if (participantInfo && participantInfo.identity === AGENT_NAME) {
|
||||
role = "agent";
|
||||
} else if (trackId) {
|
||||
for (const p of rm.remoteParticipants.values()) {
|
||||
const pub = p.getTrackPublication(trackId);
|
||||
if (pub) { role = isAgent(p) ? "agent" : "user"; break; }
|
||||
}
|
||||
}
|
||||
addMessage(role, text.trim());
|
||||
});
|
||||
} catch (e) {
|
||||
// Older SDKs may not support registerTextStreamHandler — the data
|
||||
// channel handler below still covers transcripts.
|
||||
console.warn("text stream registration failed:", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Secondary: agent publishes {type:"transcript"} on topic "transcript".
|
||||
function handleDataPacket(payload, participant, topic) {
|
||||
try {
|
||||
const msg = JSON.parse(new TextDecoder().decode(payload));
|
||||
if (msg.type === "transcript") {
|
||||
addMessage(msg.role || "agent", msg.text);
|
||||
}
|
||||
// set_voice messages flow the other way; nothing to do client-side.
|
||||
} catch (e) {
|
||||
console.warn("bad data packet", e);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Voice settings sheet ────────────────────────────────────────────────────
|
||||
function currentVoiceId() {
|
||||
const saved = localStorage.getItem("preferred_voice");
|
||||
return VOICES.some((v) => v.id === saved) ? saved : VOICES[0].id;
|
||||
}
|
||||
|
||||
function renderVoiceList() {
|
||||
voiceListEl.innerHTML = "";
|
||||
const active = currentVoiceId();
|
||||
for (const v of VOICES) {
|
||||
const card = document.createElement("div");
|
||||
card.className = "voice-card" + (v.id === active ? " active" : "");
|
||||
card.dataset.voice = v.id;
|
||||
|
||||
const name = document.createElement("span");
|
||||
name.className = "voice-name";
|
||||
name.textContent = v.name + (v.tag ? ` (${v.tag})` : "");
|
||||
|
||||
const previewBtn = document.createElement("button");
|
||||
previewBtn.className = "btn btn-ghost voice-preview";
|
||||
previewBtn.setAttribute("aria-label", `Preview ${v.name}`);
|
||||
previewBtn.title = "Preview (coming soon)";
|
||||
// TODO: play a short Azure TTS sample for this voice.
|
||||
previewBtn.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
addMessage("system", `Preview for ${v.name} is not wired up yet.`);
|
||||
});
|
||||
|
||||
card.appendChild(name);
|
||||
card.appendChild(previewBtn);
|
||||
card.addEventListener("click", () => selectVoice(v.id));
|
||||
voiceListEl.appendChild(card);
|
||||
}
|
||||
}
|
||||
|
||||
function selectVoice(voiceId) {
|
||||
localStorage.setItem("preferred_voice", voiceId);
|
||||
renderVoiceList();
|
||||
if (room && agentParticipant) {
|
||||
sendVoiceMessage(voiceId);
|
||||
const v = VOICES.find((x) => x.id === voiceId);
|
||||
addMessage("system", `Voice changed to ${v ? v.name : voiceId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function openSheet() {
|
||||
renderVoiceList();
|
||||
sheetBackdrop.hidden = false;
|
||||
settingsSheet.classList.add("open");
|
||||
}
|
||||
|
||||
function closeSheet() {
|
||||
settingsSheet.classList.remove("open");
|
||||
sheetBackdrop.hidden = true;
|
||||
}
|
||||
|
||||
settingsBtn.addEventListener("click", openSheet);
|
||||
closeSheetBtn.addEventListener("click", closeSheet);
|
||||
sheetBackdrop.addEventListener("click", closeSheet);
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape") closeSheet();
|
||||
});
|
||||
|
||||
// ── Wake lock ───────────────────────────────────────────────────────────────
|
||||
async function acquireWakeLock() {
|
||||
try {
|
||||
if ("wakeLock" in navigator) {
|
||||
wakeLock = await navigator.wakeLock.request("screen");
|
||||
}
|
||||
} catch (e) { /* not critical */ }
|
||||
}
|
||||
|
||||
function releaseWakeLock() {
|
||||
if (wakeLock) {
|
||||
wakeLock.release().catch(() => {});
|
||||
wakeLock = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Agent detection ─────────────────────────────────────────────────────────
|
||||
@@ -63,7 +460,7 @@ function markAgentFound(rm, participant) {
|
||||
agentParticipant = participant;
|
||||
statusEl.textContent = "Agent connected — speak now";
|
||||
// Send current voice selection to the agent
|
||||
sendVoiceMessage(voiceSelect.value, rm);
|
||||
sendVoiceMessage(currentVoiceId(), rm);
|
||||
}
|
||||
|
||||
function createRoom(token) {
|
||||
@@ -77,18 +474,49 @@ function createRoom(token) {
|
||||
if (isAgent(participant)) markAgentFound(newRoom, participant);
|
||||
});
|
||||
|
||||
// Listen for transcripts from the agent
|
||||
newRoom.on(RoomEvent.DataReceived, (payload) => {
|
||||
try {
|
||||
const msg = JSON.parse(payload.decodeToString());
|
||||
if (msg.type === "transcript") {
|
||||
addMessage(msg.role || "agent", msg.text);
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore non-JSON data
|
||||
// Attach and play incoming audio tracks (agent TTS)
|
||||
newRoom.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
|
||||
if (track.kind === LivekitClient.Track.Kind.Audio) {
|
||||
const el = track.attach();
|
||||
el.id = `audio-${participant.identity}`;
|
||||
document.body.appendChild(el);
|
||||
// Remote audio analyser drives the orb + visualizer while speaking
|
||||
setupRemoteAudioAnalyser(el);
|
||||
}
|
||||
});
|
||||
|
||||
newRoom.on(RoomEvent.TrackUnsubscribed, (track) => {
|
||||
track.detach().forEach((el) => el.remove());
|
||||
});
|
||||
|
||||
// Browsers block audio playback until a user gesture; show an unlock
|
||||
// button if the page loads without one.
|
||||
newRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => {
|
||||
if (!newRoom.canPlaybackAudio) {
|
||||
const existing = document.getElementById("audioUnlockBtn");
|
||||
if (!existing) {
|
||||
const btn = document.createElement("button");
|
||||
btn.id = "audioUnlockBtn";
|
||||
btn.className = "btn btn-primary";
|
||||
btn.textContent = "Tap to enable audio";
|
||||
btn.style.cssText = "position:fixed;bottom:2rem;left:50%;transform:translateX(-50%);z-index:999;padding:1rem 2rem;font-size:1.1rem;border-radius:999px;box-shadow:0 4px 24px rgba(0,0,0,.5)";
|
||||
btn.onclick = () => { newRoom.startAudio(); btn.remove(); };
|
||||
document.body.appendChild(btn);
|
||||
}
|
||||
} else {
|
||||
const existing = document.getElementById("audioUnlockBtn");
|
||||
if (existing) existing.remove();
|
||||
}
|
||||
});
|
||||
|
||||
// Live text stream (word-by-word transcripts)
|
||||
registerTranscriptionStream(newRoom);
|
||||
|
||||
// Data channel: transcripts (secondary source) + voice control
|
||||
newRoom.on(RoomEvent.DataReceived, (payload, participant, kind, topic) => {
|
||||
handleDataPacket(payload, participant, topic);
|
||||
});
|
||||
|
||||
// Guard with `=== room` so a stale (retry-discarded) connection cannot
|
||||
// clobber UI state of the active one.
|
||||
newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
|
||||
@@ -99,7 +527,11 @@ function createRoom(token) {
|
||||
|
||||
newRoom.on(RoomEvent.Disconnected, () => {
|
||||
if (newRoom !== room) return;
|
||||
teardownAudio();
|
||||
stopVisualLoop();
|
||||
releaseWakeLock();
|
||||
statusEl.textContent = "Disconnected";
|
||||
setState("idle");
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
agentParticipant = null;
|
||||
@@ -135,12 +567,15 @@ function waitForAgent(timeoutMs) {
|
||||
|
||||
startBtn.addEventListener("click", async () => {
|
||||
try {
|
||||
setState("connecting");
|
||||
statusEl.textContent = "Connecting...";
|
||||
await acquireWakeLock();
|
||||
|
||||
// Get a signed access token from the token endpoint
|
||||
const token = await fetchToken(ROOM_NAME);
|
||||
|
||||
room = await createRoom(token);
|
||||
await room.startAudio();
|
||||
|
||||
// The agent is dispatched when we join. If it misses the dispatch
|
||||
// (e.g. the server just started and the worker wasn't ready yet),
|
||||
@@ -152,7 +587,8 @@ startBtn.addEventListener("click", async () => {
|
||||
room = null; // detach first so its Disconnected handler
|
||||
agentParticipant = null; // cannot clobber the UI
|
||||
await stale.disconnect();
|
||||
room = await createRoom(token);
|
||||
const freshToken = await fetchToken(ROOM_NAME);
|
||||
room = await createRoom(freshToken);
|
||||
if (!(await waitForAgent(15000))) {
|
||||
throw new Error("Assistant did not join — reload the page and try again");
|
||||
}
|
||||
@@ -165,14 +601,14 @@ startBtn.addEventListener("click", async () => {
|
||||
startBtn.disabled = true;
|
||||
stopBtn.disabled = false;
|
||||
|
||||
// Live mic level readout: if this stays at 0 while you talk, the
|
||||
// phone is not capturing audio (iOS quirk), and the problem is on
|
||||
// the device side, not the server.
|
||||
startMicMeter();
|
||||
setupMicAnalyser();
|
||||
startVisualLoop();
|
||||
|
||||
} catch (err) {
|
||||
console.error("Connection failed:", err);
|
||||
statusEl.textContent = `Error: ${err.message}`;
|
||||
setState("idle");
|
||||
releaseWakeLock();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -183,7 +619,12 @@ stopBtn.addEventListener("click", () => {
|
||||
room = null;
|
||||
agentParticipant = null;
|
||||
}
|
||||
teardownAudio();
|
||||
stopVisualLoop();
|
||||
releaseWakeLock();
|
||||
document.querySelectorAll('[id^="audio-"]').forEach(el => el.remove());
|
||||
statusEl.textContent = "Disconnected";
|
||||
setState("idle");
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
});
|
||||
@@ -210,45 +651,3 @@ function sendVoiceMessage(voice, targetRoom) {
|
||||
reliable: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Helper: live mic level readout (diagnostic) ─────────────────────────────
|
||||
function startMicMeter() {
|
||||
const el = document.getElementById("micLevel");
|
||||
if (!el) return;
|
||||
let audioCtx = null;
|
||||
let analyser = null;
|
||||
try {
|
||||
const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
|
||||
const mst = pub && pub.track && pub.track.mediaStreamTrack;
|
||||
if (mst) {
|
||||
audioCtx = new AudioContext();
|
||||
analyser = audioCtx.createAnalyser();
|
||||
analyser.fftSize = 512;
|
||||
audioCtx.createMediaStreamSource(new MediaStream([mst])).connect(analyser);
|
||||
}
|
||||
} catch (e) { /* fall back to SDK audioLevel below */ }
|
||||
|
||||
const buf = analyser ? new Float32Array(analyser.fftSize) : null;
|
||||
window.micMeterTimer = setInterval(() => {
|
||||
if (!room || !room.localParticipant) { clearInterval(window.micMeterTimer); return; }
|
||||
let level = 0;
|
||||
if (analyser) {
|
||||
analyser.getFloatTimeDomainData(buf);
|
||||
for (let i = 0; i < buf.length; i++) level = Math.max(level, Math.abs(buf[i]));
|
||||
} else {
|
||||
level = room.localParticipant.audioLevel;
|
||||
}
|
||||
el.textContent = `mic: ${Math.round(level * 100)}%`;
|
||||
el.style.color = level > 0.02 ? "#4ade80" : "#94a3b8";
|
||||
}, 300);
|
||||
}
|
||||
|
||||
// ── Helper: add a message to the transcript ─────────────────────────────────
|
||||
function addMessage(role, text) {
|
||||
const div = document.createElement("div");
|
||||
div.className = `message ${role}`;
|
||||
const label = role === "user" ? "You" : role === "agent" ? "Assistant" : "System";
|
||||
div.innerHTML = `<span class="role">${label}</span> ${text}`;
|
||||
messagesEl.appendChild(div);
|
||||
messagesEl.scrollTop = messagesEl.scrollHeight;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user