feat: media player API, web player panel, and Hope music MCP

- web/media_server.py: FastAPI service owning the permanent library
  (library.json + ./media volume), yt-dlp YouTube Music search, queued
  download jobs with mp3 transcode (ffmpeg), per-room player desired
  state, and an HTTPS proxy for external live-stream URLs.
- agent/media_mcp.py: Hope tools — search_music, play_song,
  offer_song_choices, pause/resume/stop/skip/seek, now_playing,
  list_library, add_stream.
- agent/agent.py: media toolset wiring + conversation rule (music
  playing disables the agent mic; re-engaging pauses music).
- web UI: slide-out Music panel (search, suggestions, library, live
  stream add, play/pause, ±10s, seek, volume) with a reconciling poller
  and media_state data-channel reports.
- token server returns the minted room name; nginx serves /media/files
  as static (native Range) and proxies /media/api to the media server.
- supervisord runs media-server; compose mounts ./media and adds
  ffmpeg to the image.
This commit is contained in:
snowspeeder committed 2026-08-29 13:03:12 -04:00
1 parent a4ae96f01b
commit 04c4a53594
16 files changed
+1876 -12

No files matched your search

+511 -5
View File
@@ -693,6 +693,7 @@ document.addEventListener("keydown", (event) => {
if (event.key !== "Escape") return;
if (settingsSheet.classList.contains("open")) closeSheet();
if (tasksPanel.classList.contains("open")) setTasksOpen(false);
if (mediaPanel.classList.contains("open")) setMediaOpen(false);
});
// ── Wake lock ───────────────────────────────────────────────────────────────
async function acquireWakeLock() {
@@ -816,6 +817,8 @@ function createRoom(token) {
stopBtn.disabled = true;
agentParticipant = null;
agentReady = false;
// Leaving the conversation hands the room back to the music player.
resumePlaybackAfterTalk();
});
return newRoom.connect(LIVEKIT_URL, token, {
@@ -873,8 +876,9 @@ async function connectConversation() {
await acquireWakeLock();
// Get a signed access token from the token endpoint.
const token = await fetchToken(ROOM_NAME);
room = await createRoom(token);
const tokenData = await fetchToken(ROOM_NAME);
await enterMediaScope(tokenData.room);
room = await createRoom(tokenData.token);
// The agent is dispatched when we join. If it misses the dispatch
// (for example during a server restart), reconnect once.
@@ -886,8 +890,9 @@ async function connectConversation() {
agentParticipant = null;
agentReady = false;
await stale.disconnect();
const freshToken = await fetchToken(ROOM_NAME);
room = await createRoom(freshToken);
const freshTokenData = await fetchToken(ROOM_NAME);
await enterMediaScope(freshTokenData.room);
room = await createRoom(freshTokenData.token);
if (!(await waitForAgent(15000))) {
throw new Error("Assistant did not join — reload the page and try again");
}
@@ -999,6 +1004,8 @@ stopBtn.addEventListener("click", () => {
muteBtn.disabled = true;
micMuted = false;
updateMuteIcon();
// Ending the conversation resumes music that was paused for the talk.
resumePlaybackAfterTalk();
});
// ── Mic mute toggle ─────────────────────────────────────────────────────────
@@ -1030,7 +1037,7 @@ async function fetchToken(roomName) {
});
if (!resp.ok) throw new Error(`Token endpoint returned ${resp.status}`);
const data = await resp.json();
return data.token;
return { token: data.token, room: data.room || null };
}
// ── Helper: send voice selection to the agent via data channel ─────────────
@@ -1064,3 +1071,502 @@ function sendModelMessage(model, targetRoom) {
reliable: true,
});
}
// ── Media player ────────────────────────────────────────────────────────────
// The browser <audio> element is the player. The media server owns library +
// per-room desired state; this UI reconciles against GET /media/api/state
// (polling) and reports real playback back. Hope drives the same state via
// her media tools, so voice commands and panel clicks stay in sync.
//
// Conversation rule (mirrored in agent.py): while a track plays the agent's
// mic is off. Connecting to Hope pauses music; ending the conversation
// resumes what was playing.
const mediaBtn = document.getElementById("mediaBtn");
const mediaPanel = document.getElementById("mediaPanel");
const closeMediaBtn = document.getElementById("closeMediaBtn");
const mediaBadge = document.getElementById("mediaBadge");
const mediaArt = document.getElementById("mediaArt");
const mediaArtPlaceholder = document.getElementById("mediaArtPlaceholder");
const mediaTitleEl = document.getElementById("mediaTitle");
const mediaChannelEl = document.getElementById("mediaChannel");
const mediaProgressWrap = document.getElementById("mediaProgressWrap");
const mediaProgressFill = document.getElementById("mediaProgressFill");
const mediaProgressLabel = document.getElementById("mediaProgressLabel");
const rewBtn = document.getElementById("rewBtn");
const fwdBtn = document.getElementById("fwdBtn");
const playPauseBtn = document.getElementById("playPauseBtn");
const playIcon = document.getElementById("playIcon");
const pauseIcon = document.getElementById("pauseIcon");
const stopMediaBtn = document.getElementById("stopMediaBtn");
const seekBar = document.getElementById("seekBar");
const mediaCurEl = document.getElementById("mediaCur");
const mediaDurEl = document.getElementById("mediaDur");
const volumeBar = document.getElementById("volumeBar");
const mediaSearchForm = document.getElementById("mediaSearchForm");
const mediaSearchInput = document.getElementById("mediaSearchInput");
const mediaResultsEl = document.getElementById("mediaResults");
const mediaChoicesWrap = document.getElementById("mediaChoicesWrap");
const mediaChoicesEl = document.getElementById("mediaChoices");
const mediaLibraryEl = document.getElementById("mediaLibrary");
const streamForm = document.getElementById("streamForm");
const streamNameInput = document.getElementById("streamName");
const streamUrlInput = document.getElementById("streamUrl");
const mediaAudio = document.getElementById("mediaAudio");
let mediaRoom = null; // LiveKit room name (player state scope)
let mediaCurrentTrackId = null; // track id loaded into the audio element
let mediaAppliedSeekTs = 0; // last applied seek_hint timestamp
let mediaWasPlayingBeforeTalk = false;
let mediaPollTimer = null;
let mediaLastReport = 0;
function mediaScope() {
return mediaRoom || "default";
}
async function mediaPost(path, body) {
const resp = await fetch(`/media/api${path}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ room: mediaScope(), ...body }),
});
if (!resp.ok) {
const detail = await resp.json().catch(() => ({}));
throw new Error(detail.error || `media API ${path} -> ${resp.status}`);
}
return resp.json();
}
// Called just before joining the LiveKit room: carry whatever the audio
// element holds into the room-scoped player state, paused (talking pauses
// music). resumePlaybackAfterTalk() reverses this when the talk ends.
async function enterMediaScope(newRoom) {
if (!newRoom || newRoom === mediaRoom) return;
const wasPlaying = !!mediaCurrentTrackId && !mediaAudio.paused;
mediaWasPlayingBeforeTalk = wasPlaying;
const carriedTrack = mediaCurrentTrackId;
mediaRoom = newRoom;
if (carriedTrack) {
try {
await mediaPost("/play", { room: mediaRoom, track_id: carriedTrack, start_paused: true });
} catch (e) { console.warn("media scope transfer failed:", e); }
mediaAudio.pause();
}
}
async function resumePlaybackAfterTalk() {
if (!mediaWasPlayingBeforeTalk) return;
mediaWasPlayingBeforeTalk = false;
try { await mediaPost("/control", { action: "resume" }); } catch (e) { /* poll retries */ }
}
function publishMediaState(state) {
if (!room || !room.localParticipant) return;
const payload = new TextEncoder().encode(
JSON.stringify({ type: "media_state", state, track_id: mediaCurrentTrackId })
);
room.localParticipant.publishData(payload, { topic: "media-control", reliable: true });
}
function reportMediaPlayback(state) {
const body = {
state,
track_id: mediaCurrentTrackId,
position: isFinite(mediaAudio.currentTime) ? mediaAudio.currentTime : 0,
};
if (isFinite(mediaAudio.duration)) body.duration = mediaAudio.duration;
mediaPost("/report", body).catch(() => {});
}
// ── Polling + reconciliation ────────────────────────────────────────────────
function startMediaPolling() {
if (mediaPollTimer) return;
mediaPollTimer = setInterval(() => {
pollMediaState();
}, 1500);
}
async function pollMediaState() {
let s;
try {
const resp = await fetch(`/media/api/state?room=${encodeURIComponent(mediaScope())}`);
if (!resp.ok) return;
s = await resp.json();
} catch (e) {
return; // media server briefly unavailable; try again next tick
}
reconcileMedia(s);
}
function reconcileMedia(s) {
renderMediaChoices(s.choices || []);
updateMediaBadge(s);
const track = s.track;
const pendingJob = (s.pending || []).find(j => !track || j.track_id === track.id);
updateMediaProgress(track, pendingJob);
if (track && track.id !== mediaCurrentTrackId) loadMediaTrack(track);
if (!track && mediaCurrentTrackId && s.desired === "stopped") clearMediaAudio();
applyDesiredMediaState(s, track);
if (s.seek_hint && s.seek_hint.ts !== mediaAppliedSeekTs && isFinite(mediaAudio.duration)) {
mediaAppliedSeekTs = s.seek_hint.ts;
const target = Math.min(s.seek_hint.position, mediaAudio.duration);
if (Number.isFinite(target)) {
mediaAudio.currentTime = Math.max(0, target);
updateSeekUI();
}
}
}
function applyDesiredMediaState(s, track) {
if (s.desired === "playing") {
if (track && track.status === "ready" && mediaAudio.src && mediaAudio.paused) {
mediaAudio.play().catch((e) => console.warn("playback blocked:", e));
}
} else if (!mediaAudio.paused) {
mediaAudio.pause();
}
}
function loadMediaTrack(track) {
mediaCurrentTrackId = track.id;
updateMediaNowCard(track);
if (track.audio_url && (track.kind === "stream" || track.status === "ready")) {
if (mediaAudio.getAttribute("src") !== track.audio_url) {
mediaAudio.src = track.audio_url;
mediaAudio.load();
}
} else {
// Downloading or failed: no stream yet. The poll flips this to the
// real URL once status turns "ready".
mediaAudio.removeAttribute("src");
mediaAudio.load();
updateMediaControls();
}
}
function clearMediaAudio() {
mediaCurrentTrackId = null;
mediaAudio.pause();
mediaAudio.removeAttribute("src");
mediaAudio.load();
mediaTitleEl.textContent = "Nothing loaded";
mediaChannelEl.textContent = "";
mediaArt.hidden = true;
mediaArtPlaceholder.hidden = false;
updateMediaControls();
}
function updateMediaNowCard(track) {
mediaTitleEl.textContent = track.title || track.id;
mediaChannelEl.textContent = track.channel || (track.kind === "stream" ? "Live stream" : "");
if (track.art_url) {
mediaArt.src = track.art_url;
mediaArt.hidden = false;
mediaArtPlaceholder.hidden = true;
} else {
mediaArt.hidden = true;
mediaArtPlaceholder.hidden = false;
}
}
function updateMediaProgress(track, job) {
if (!job) {
mediaProgressWrap.hidden = true;
return;
}
mediaProgressWrap.hidden = false;
const pct = Math.round(job.progress || 0);
mediaProgressFill.style.width = `${pct}%`;
const stage = job.stage === "transcoding"
? "Re-encoding…"
: job.stage === "queued" ? "Queued…" : "Downloading…";
mediaProgressLabel.textContent = `${stage} ${pct}%`;
if (track) updateMediaNowCard(track);
}
function updateMediaBadge(s) {
const pending = (s.pending || []).length;
const playing = !mediaAudio.paused && !!mediaCurrentTrackId;
if (pending > 0) {
mediaBadge.hidden = false;
mediaBadge.textContent = String(pending);
} else if (playing) {
mediaBadge.hidden = false;
mediaBadge.innerHTML = "&#9835;";
} else {
mediaBadge.hidden = true;
}
}
// ── Playback controls ───────────────────────────────────────────────────────
function updateMediaControls() {
const playing = !mediaAudio.paused && !!mediaCurrentTrackId;
playIcon.hidden = playing;
pauseIcon.hidden = !playing;
playPauseBtn.setAttribute("aria-label", playing ? "Pause" : "Play");
updateMediaBadge({ pending: [] });
}
function fmtTime(sec) {
if (!isFinite(sec) || sec < 0) return "0:00";
const s = Math.floor(sec % 60);
const m = Math.floor((sec / 60) % 60);
const h = Math.floor(sec / 3600);
return h ? `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}` : `${m}:${String(s).padStart(2, "0")}`;
}
function updateSeekUI() {
if (!isFinite(mediaAudio.duration) || mediaAudio.duration <= 0) {
seekBar.value = 0;
mediaDurEl.textContent = "0:00";
mediaCurEl.textContent = fmtTime(mediaAudio.currentTime);
return;
}
seekBar.max = Math.floor(mediaAudio.duration);
seekBar.value = Math.floor(mediaAudio.currentTime);
mediaCurEl.textContent = fmtTime(mediaAudio.currentTime);
mediaDurEl.textContent = fmtTime(mediaAudio.duration);
}
playPauseBtn.addEventListener("click", () => {
if (!mediaCurrentTrackId || !mediaAudio.src) return;
if (mediaAudio.paused) {
mediaAudio.play().catch((e) => console.warn("play failed:", e));
mediaPost("/control", { action: "resume" }).catch(() => {});
} else {
mediaAudio.pause();
mediaPost("/control", { action: "pause" }).catch(() => {});
}
});
rewBtn.addEventListener("click", () => {
if (!mediaAudio.src) return;
mediaAudio.currentTime = Math.max(0, mediaAudio.currentTime - 10);
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
});
fwdBtn.addEventListener("click", () => {
if (!mediaAudio.src || !isFinite(mediaAudio.duration)) return;
mediaAudio.currentTime = Math.min(mediaAudio.duration - 0.5, mediaAudio.currentTime + 10);
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
});
stopMediaBtn.addEventListener("click", () => {
clearMediaAudio();
mediaPost("/control", { action: "stop" }).catch(() => {});
publishMediaState("stopped");
});
seekBar.addEventListener("change", () => {
if (!mediaAudio.src || !isFinite(mediaAudio.duration)) return;
mediaAudio.currentTime = Number(seekBar.value);
reportMediaPlayback(!mediaAudio.paused ? "playing" : "paused");
});
volumeBar.addEventListener("input", () => {
mediaAudio.volume = Number(volumeBar.value) / 100;
try { localStorage.setItem("media_volume", volumeBar.value); } catch (e) { /* ok */ }
});
mediaAudio.volume = Number(localStorage.getItem("media_volume") ?? 90) / 100;
volumeBar.value = Math.round(mediaAudio.volume * 100);
mediaAudio.addEventListener("play", () => {
updateMediaControls();
reportMediaPlayback("playing");
publishMediaState("playing");
});
mediaAudio.addEventListener("pause", () => {
updateMediaControls();
if (mediaCurrentTrackId) {
reportMediaPlayback("paused");
publishMediaState("paused");
}
});
mediaAudio.addEventListener("ended", () => {
updateMediaControls();
reportMediaPlayback("stopped");
publishMediaState("stopped");
mediaPost("/control", { action: "stop" }).catch(() => {});
});
mediaAudio.addEventListener("timeupdate", () => {
updateSeekUI();
const now = Date.now();
if (!mediaAudio.paused && now - mediaLastReport > 5000) {
mediaLastReport = now;
reportMediaPlayback("playing");
}
});
mediaAudio.addEventListener("durationchange", updateSeekUI);
mediaAudio.addEventListener("error", () => {
if (mediaCurrentTrackId) {
mediaChannelEl.textContent = "Playback error — the track may still be preparing.";
}
});
// ── Search / choices / library ──────────────────────────────────────────────
function fmtListDuration(sec) {
const s = Number(sec);
return isFinite(s) && s > 0 ? fmtTime(s) : "";
}
function mediaRowActions() {
const play = document.createElement("button");
play.className = "btn btn-secondary media-row-play";
play.type = "button";
play.textContent = "Play";
return play;
}
function renderMediaResults(results) {
mediaResultsEl.innerHTML = "";
if (!results || results.length === 0) {
mediaResultsEl.innerHTML = '<div class="media-empty">No results.</div>';
return;
}
for (const r of results) {
const row = document.createElement("div");
row.className = "media-row";
const info = document.createElement("div");
info.className = "media-row-info";
const dur = fmtListDuration(r.duration);
info.innerHTML = `<strong>${escapeHtml(r.title)}</strong>` +
`<span>${escapeHtml(r.channel || "")}${dur ? " · " + dur : ""}` +
`${r.in_library ? " · in library" : ""}</span>`;
const play = mediaRowActions();
play.addEventListener("click", () => playVideoId(r));
row.appendChild(info);
row.appendChild(play);
mediaResultsEl.appendChild(row);
}
}
async function playVideoId(r) {
try {
await mediaPost("/play", {
video_id: r.video_id,
title: r.title,
channel: r.channel || "",
});
} catch (e) {
mediaChannelEl.textContent = `Could not start: ${e.message}`;
}
}
mediaSearchForm.addEventListener("submit", async (event) => {
event.preventDefault();
const query = mediaSearchInput.value.trim();
if (!query) return;
mediaResultsEl.innerHTML = '<div class="media-empty">Searching YouTube Music…</div>';
try {
const data = await mediaPost("/search", { query, limit: 6 });
renderMediaResults(data.results);
} catch (e) {
mediaResultsEl.innerHTML = `<div class="media-empty">Search failed: ${escapeHtml(e.message)}</div>`;
}
});
function renderMediaChoices(choices) {
mediaChoicesWrap.hidden = choices.length === 0;
mediaChoicesEl.innerHTML = "";
for (const c of choices) {
const row = document.createElement("div");
row.className = "media-row media-choice";
const info = document.createElement("div");
info.className = "media-row-info";
const dur = fmtListDuration(c.duration);
info.innerHTML = `<strong>${escapeHtml(c.title)}</strong>` +
`<span>${escapeHtml(c.channel || "")}${dur ? " · " + dur : ""}</span>`;
const play = mediaRowActions();
play.addEventListener("click", () => playVideoId(c));
row.appendChild(info);
row.appendChild(play);
mediaChoicesEl.appendChild(row);
}
}
async function refreshLibrary() {
let data;
try {
data = await fetch("/media/api/library").then((r) => r.json());
} catch (e) {
return;
}
mediaLibraryEl.innerHTML = "";
const tracks = data.tracks || [];
if (tracks.length === 0) {
mediaLibraryEl.innerHTML = '<div class="media-empty">Downloaded songs and streams live here.</div>';
return;
}
for (const t of tracks) {
const row = document.createElement("div");
row.className = "media-row";
const info = document.createElement("div");
info.className = "media-row-info";
const dur = fmtListDuration(t.duration);
const statusTag = t.status === "ready" ? "" : ` · ${t.status}`;
info.innerHTML = `<strong>${escapeHtml(t.title)}</strong>` +
`<span>${escapeHtml(t.channel || (t.kind === "stream" ? "Live stream" : ""))}` +
`${dur ? " · " + dur : ""}${statusTag}</span>`;
const play = mediaRowActions();
play.disabled = t.status !== "ready";
play.addEventListener("click", () => {
mediaPost("/play", { track_id: t.id }).catch((e) => {
mediaChannelEl.textContent = `Could not play: ${e.message}`;
});
});
const del = document.createElement("button");
del.className = "btn btn-icon media-row-delete";
del.type = "button";
del.setAttribute("aria-label", `Remove ${t.title}`);
del.title = "Remove from library";
del.innerHTML = '<svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><path d="M18 6 6 18M6 6l12 12"></path></svg>';
del.addEventListener("click", async () => {
try {
await fetch(`/media/api/tracks/${encodeURIComponent(t.id)}`, { method: "DELETE" });
if (t.id === mediaCurrentTrackId) clearMediaAudio();
refreshLibrary();
} catch (e) { /* leave the row; next refresh decides */ }
});
row.appendChild(info);
row.appendChild(play);
row.appendChild(del);
mediaLibraryEl.appendChild(row);
}
}
streamForm.addEventListener("submit", async (event) => {
event.preventDefault();
const name = streamNameInput.value.trim() || streamUrlInput.value.trim();
const url = streamUrlInput.value.trim();
if (!url) return;
try {
await mediaPost("/streams", { name, url });
streamNameInput.value = "";
streamUrlInput.value = "";
refreshLibrary();
} catch (e) {
mediaChannelEl.textContent = `Could not add stream: ${e.message}`;
}
});
// ── Panel open/close ────────────────────────────────────────────────────────
function setMediaOpen(open) {
mediaPanel.classList.toggle("open", open);
mediaBtn.setAttribute("aria-expanded", String(open));
if (open) {
refreshLibrary();
pollMediaState();
}
}
mediaBtn.setAttribute("aria-expanded", "false");
mediaBtn.addEventListener("click", () => setMediaOpen(!mediaPanel.classList.contains("open")));
closeMediaBtn.addEventListener("click", () => setMediaOpen(false));
startMediaPolling();
updateMediaControls();