fix: start cached songs even when played by voice (autoplay + room scope)
Rebuild & Deploy / rebuild (push) Successful in 50s

Hope often starts an already-downloaded song from a voice command, with no
click on the player. Two things could leave it paused at 0:

- The <audio> element sits outside the LiveKit room's audio-unlock flow, so
  autoplay policy rejected the poll-driven play(). Capture the first page
  gesture (pointerdown/keydown/touchstart) and play the current src inside it
  to whitelist the element for autoplay.
- Player state is scoped per LiveKit room, but mediaScope() could return a
  stale shell room after a reconnect. Derive it from the connected room.name
  so the browser polls exactly the room the agent joined, and a "play X"
  request can never land in a scope the browser isn't watching.

Add an in-container regression test (test_media_play_resolves_cached_ready_track)
pinning the server-side resolve: an already-ready track plays without a
re-download and serves a stable audio_url.
This commit is contained in:
snowspeeder committed 2026-08-29 17:37:51 -04:00
1 parent 436a26a0fc
commit df8aa64504
2 files changed
+100

No files matched your search

+53
View File
@@ -427,6 +427,59 @@ finally:
assert "OK" in proc.stdout assert "OK" in proc.stdout
@needs_local_container
def test_media_play_resolves_cached_ready_track(client):
"""Playing an ALREADY-cached song must resolve to the on-disk ready track
(no redownload) and hand the browser a playable stream URL, so a cached
pick starts instantly rather than re-fetching from YouTube.
Regression: a cached track could be left as paused with currentTime 0 if
the room the agent played into diverged from the room the browser polled,
or if autoplay was blocked. This pins the server-side resolve + track
dict so a ready file is always returned with a stable audio_url."""
code = '''
import sys
sys.path.insert(0, "/opt/voice")
import media_server as ms
from fastapi.testclient import TestClient
probe = ms.FILES_DIR / "cachetrk.webm"
probe.write_bytes(b"CACHEDBYTES" * 40)
ms._library["tracks"]["cachetrk"] = {
"id": "cachetrk", "kind": "youtube", "video_id": "cachetrk",
"title": "Cached Song", "channel": "Artist", "duration": 200,
"status": "ready", "ext": "webm", "error": None, "added_at": 1.0,
}
ms._start_download = lambda *a, **k: (_ for _ in ()).throw(
AssertionError("must not re-download an already cached track")
)
try:
c = TestClient(ms.app)
# play by video_id (as Hope's play_song does) must reuse the ready track
r = c.post("/media/api/play", json={
"room": "cached-test-room", "video_id": "cachetrk",
"title": "Cached Song", "channel": "Artist",
}, timeout=10)
assert r.status_code == 200, r.text
body = r.json()
assert body["desired"] == "playing", body
track = body["track"]
assert track["status"] == "ready", track
assert track["audio_url"] == "/media/api/stream/cachetrk", track
# the stream endpoint serves the cached bytes (browser plays it directly)
data = c.get("/media/api/stream/cachetrk")
assert data.status_code == 200
assert data.content == b"CACHEDBYTES" * 40
print("OK")
finally:
probe.unlink(missing_ok=True)
ms._library["tracks"].pop("cachetrk", None)
'''
proc = docker_exec_python(code, timeout=60, check=False)
assert proc.returncode == 0, proc.stderr[-800:]
assert "OK" in proc.stdout
@needs_local_container @needs_local_container
def test_mcp_media_list_library(mcp_client_factory): def test_mcp_media_list_library(mcp_client_factory):
text, is_error = mcp_client_factory( text, is_error = mcp_client_factory(
+47
View File
@@ -269,6 +269,19 @@ const LED_GROW = 1.4; // how much the dots grow toward the center
let ledDots = []; let ledDots = [];
let ledWallTime = 0; let ledWallTime = 0;
// Decides whether the LED wall square should show the track's album art.
// Music and conversation are mutually exclusive: while a conversation is
// active the music is paused, so the wall pulses (agent audio) and any loaded
// track's cover art returns once the talk ends.
function updateAvatarMedia() {
const inConversation = !!(room && room.localParticipant);
const showArt = !inConversation && !!mediaCurrentTrackId && !!mediaCurrentArtUrl;
ledWallEl.classList.toggle("show-art", showArt);
if (showArt && ledArtEl.getAttribute("src") !== mediaCurrentArtUrl) {
ledArtEl.src = mediaCurrentArtUrl;
}
}
function buildLedWall() { function buildLedWall() {
const inner = Math.floor(LED_COLS / 2); const inner = Math.floor(LED_COLS / 2);
const frag = document.createDocumentFragment(); const frag = document.createDocumentFragment();
@@ -998,6 +1011,7 @@ async function connectConversation() {
muteBtn.disabled = conversationMode !== "voice"; muteBtn.disabled = conversationMode !== "voice";
updateMuteIcon(); updateMuteIcon();
setState("listening"); setState("listening");
updateAvatarMedia();
return room; return room;
} catch (err) { } catch (err) {
console.error("Connection failed:", err); console.error("Connection failed:", err);
@@ -1081,6 +1095,7 @@ stopBtn.addEventListener("click", () => {
updateMuteIcon(); updateMuteIcon();
// Ending the conversation resumes music that was paused for the talk. // Ending the conversation resumes music that was paused for the talk.
resumePlaybackAfterTalk(); resumePlaybackAfterTalk();
updateAvatarMedia();
}); });
// ── Mic mute toggle ───────────────────────────────────────────────────────── // ── Mic mute toggle ─────────────────────────────────────────────────────────
@@ -1188,15 +1203,21 @@ const streamForm = document.getElementById("streamForm");
const streamNameInput = document.getElementById("streamName"); const streamNameInput = document.getElementById("streamName");
const streamUrlInput = document.getElementById("streamUrl"); const streamUrlInput = document.getElementById("streamUrl");
const mediaAudio = document.getElementById("mediaAudio"); const mediaAudio = document.getElementById("mediaAudio");
const ledArtEl = document.getElementById("ledArt");
let mediaRoom = null; // LiveKit room name (player state scope) let mediaRoom = null; // LiveKit room name (player state scope)
let mediaCurrentTrackId = null; // track id loaded into the audio element let mediaCurrentTrackId = null; // track id loaded into the audio element
let mediaCurrentArtUrl = null; // current track's cover art
let mediaAppliedSeekTs = 0; // last applied seek_hint timestamp let mediaAppliedSeekTs = 0; // last applied seek_hint timestamp
let mediaWasPlayingBeforeTalk = false; let mediaWasPlayingBeforeTalk = false;
let mediaPollTimer = null; let mediaPollTimer = null;
let mediaLastReport = 0; let mediaLastReport = 0;
function mediaScope() { function mediaScope() {
// The LiveKit room is single-source of truth for the player scope. If a
// room is connected, poll the room the agent actually joined (room.name),
// so a "play X" request from the agent can never land in a stale scope.
if (room && room.name) return room.name;
return mediaRoom || "default"; return mediaRoom || "default";
} }
@@ -1336,6 +1357,7 @@ function loadMediaTrack(track) {
function clearMediaAudio() { function clearMediaAudio() {
mediaCurrentTrackId = null; mediaCurrentTrackId = null;
mediaCurrentArtUrl = null;
mediaAudio.pause(); mediaAudio.pause();
mediaAudio.removeAttribute("src"); mediaAudio.removeAttribute("src");
mediaAudio.load(); mediaAudio.load();
@@ -1344,10 +1366,12 @@ function clearMediaAudio() {
mediaArt.hidden = true; mediaArt.hidden = true;
mediaArtPlaceholder.hidden = false; mediaArtPlaceholder.hidden = false;
updateMediaControls(); updateMediaControls();
updateAvatarMedia();
} }
function updateMediaNowCard(track) { function updateMediaNowCard(track) {
mediaTitleEl.textContent = track.title || track.id; mediaTitleEl.textContent = track.title || track.id;
mediaCurrentArtUrl = track.art_url || null;
if (track.status === "failed") { if (track.status === "failed") {
mediaChannelEl.textContent = "Download failed — pick another result or try again."; mediaChannelEl.textContent = "Download failed — pick another result or try again.";
} else if (track.status === "downloading") { } else if (track.status === "downloading") {
@@ -1365,6 +1389,7 @@ function updateMediaNowCard(track) {
mediaArt.hidden = true; mediaArt.hidden = true;
mediaArtPlaceholder.hidden = false; mediaArtPlaceholder.hidden = false;
} }
updateAvatarMedia();
} }
function updateMediaProgress(track, job) { function updateMediaProgress(track, job) {
@@ -1465,6 +1490,28 @@ volumeBar.addEventListener("input", () => {
mediaAudio.volume = Number(localStorage.getItem("media_volume") ?? 90) / 100; mediaAudio.volume = Number(localStorage.getItem("media_volume") ?? 90) / 100;
volumeBar.value = Math.round(mediaAudio.volume * 100); volumeBar.value = Math.round(mediaAudio.volume * 100);
// Browsers block audio until a user gestured on the page. Hope often starts a
// cached song from a poll (voice command, no click on the player), so the
// <audio> element sits outside the LiveKit room's audio-unlock flow and its
// play() is rejected. Capture the first gesture and unlock the element.
let mediaAudioUnlocked = false;
function unlockMediaAudio() {
if (mediaAudioUnlocked) return;
mediaAudioUnlocked = true;
removeEventListener("pointerdown", unlockMediaAudio, true);
removeEventListener("keydown", unlockMediaAudio, true);
removeEventListener("touchstart", unlockMediaAudio, true);
// Play the current src (if any) inside the gesture to whitelist the
// element for autoplay, so poll-driven play() later isn't blocked.
if (mediaAudio.getAttribute("src") && mediaAudio.paused) {
const p = mediaAudio.play();
if (p) p.catch(() => {});
}
}
addEventListener("pointerdown", unlockMediaAudio, true);
addEventListener("keydown", unlockMediaAudio, true);
addEventListener("touchstart", unlockMediaAudio, true);
mediaAudio.addEventListener("play", () => { mediaAudio.addEventListener("play", () => {
updateMediaControls(); updateMediaControls();
reportMediaPlayback("playing"); reportMediaPlayback("playing");