feat: media player API, web player panel, and Hope music MCP

- web/media_server.py: FastAPI service owning the permanent library
  (library.json + ./media volume), yt-dlp YouTube Music search, queued
  download jobs with mp3 transcode (ffmpeg), per-room player desired
  state, and an HTTPS proxy for external live-stream URLs.
- agent/media_mcp.py: Hope tools — search_music, play_song,
  offer_song_choices, pause/resume/stop/skip/seek, now_playing,
  list_library, add_stream.
- agent/agent.py: media toolset wiring + conversation rule (music
  playing disables the agent mic; re-engaging pauses music).
- web UI: slide-out Music panel (search, suggestions, library, live
  stream add, play/pause, ±10s, seek, volume) with a reconciling poller
  and media_state data-channel reports.
- token server returns the minted room name; nginx serves /media/files
  as static (native Range) and proxies /media/api to the media server.
- supervisord runs media-server; compose mounts ./media and adds
  ffmpeg to the image.

agent/agent.py

@@ -196,6 +196,23 @@ SYSTEM_PROMPT = textwrap.dedent("""\
     you receive it, share the findings naturally in your conversational style.
     You can have multiple tasks running at once.
 
+    # Music (media player in the web UI)
+    You can play music on the user's web player. For any music request, call
+    search_music first, then:
+    - One clearly-right result → play_song with its video_id. Announce it in
+      one short sentence, then stop talking — playback pauses the
+      conversation until the user comes back to you.
+    - Ambiguous results → offer_song_choices with the top 2-3 options, name
+      them briefly, and let the user pick by voice or in the player panel.
+      Never start playback on a guess.
+    The library is permanent: list_library shows what is already downloaded,
+    and those tracks play instantly without searching. If the user names a
+    library track exactly, just play it. Control playback with pause_music,
+    resume_music, skip_song and seek_music; check what's on with now_playing.
+    add_stream registers a live radio or direct audio URL as a library track.
+    While music is playing you cannot hear the user — they will come back to
+    you when they want to talk again, and the music pauses automatically.
+
     # Skills
     You have a skill library of learned procedures. Use skill_recall when you need
     to perform a task you've done before — it will give you the steps. When you
@@ -624,6 +641,27 @@ def build_mcp_toolsets(user_profile: dict, task_dir: str) -> list[mcp.MCPToolset
     )
     logger.info("Dispatch MCP toolset enabled (%s)", task_dir)
 
+    # Media tools talk to the local media server; the room name scopes player
+    # state so concurrent sessions don't fight over one virtual player.
+    media_server_url = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092")
+    media_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media_mcp.py")
+    toolsets.append(
+        mcp.MCPToolset(
+            id="media",
+            mcp_server=mcp.MCPServerStdio(
+                command=python_bin,
+                args=[media_mcp_script],
+                env={
+                    **os.environ,
+                    "MEDIA_SERVER_URL": media_server_url,
+                    "MEDIA_ROOM": os.path.basename(task_dir.rstrip("/")) or "default",
+                },
+                client_session_timeout_seconds=60,
+            ),
+        )
+    )
+    logger.info("Media MCP toolset enabled (%s)", media_server_url)
+
     extra = os.environ.get("EXTRA_MCP_SERVERS", "")
     if extra:
         try:
@@ -740,9 +778,22 @@ async def handle_job(ctx: JobContext) -> None:
 
     # Listen for conversation controls (voice selection and voice/text mode).
     # Defined here so it captures this job's session via closure.
+    # Music and conversation are mutually exclusive by design: while a track
+    # plays the agent's ears are off; re-engaging the conversation pauses the
+    # music, and ending the conversation lets the user resume it.
+    session_mode = {"value": "voice"}       # UI voice/text switch
+    media_playback = {"value": "stopped"}   # UI-reported player state
+
+    def _apply_listening() -> None:
+        listening = (
+            session_mode["value"] == "voice"
+            and media_playback["value"] != "playing"
+        )
+        session.input.set_audio_enabled(listening)
+
     def _on_room_data(packet) -> None:
         topic = getattr(packet, "topic", None)
-        if topic and topic not in ("voice-control", "conversation-control"):
+        if topic and topic not in ("voice-control", "conversation-control", "media-control"):
             return
         try:
             msg = json.loads(packet.data.decode("utf-8"))
@@ -755,11 +806,22 @@ async def handle_job(ctx: JobContext) -> None:
                 return
             voice_enabled = mode == "voice"
             logger.info("Switching conversation mode to %s", mode)
+            session_mode["value"] = mode
             if not voice_enabled:
                 # interrupt() returns an already-scheduled Future, not a coroutine.
                 session.interrupt()
-            session.input.set_audio_enabled(voice_enabled)
             session.output.set_audio_enabled(voice_enabled)
+            _apply_listening()
+            return
+
+        if msg.get("type") == "media_state":
+            state = msg.get("state")
+            if state not in ("playing", "paused", "stopped"):
+                return
+            logger.info("Media playback -> %s (agent listening: %s)",
+                        state, session_mode["value"] == "voice" and state != "playing")
+            media_playback["value"] = state
+            _apply_listening()
             return
 
         if msg.get("type") == "set_voice":

agent/media_mcp.py

@@ -0,0 +1,229 @@
+"""Media MCP server — lets Hope find and control music by voice.
+
+Tools talk to the local media server (web/media_server.py) over HTTP, which
+owns the library, YouTube Music search/download jobs and per-room player
+state. The browser is the actual player: it reconciles against the media
+server and reports playback back, so voice controls and the web panel stay
+in sync.
+
+Per-session environment (set by the agent when spawning this MCP):
+- MEDIA_SERVER_URL  e.g. http://127.0.0.1:8092
+- MEDIA_ROOM        LiveKit room name this session controls
+
+The "stop listening while music plays" rule lives in the agent itself: the
+UI reports media_state over the data channel and the agent disables its mic
+while a track is playing.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+import httpx
+from mcp.server.fastmcp import FastMCP
+
+MEDIA_SERVER_URL = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092").rstrip("/")
+MEDIA_ROOM = os.environ.get("MEDIA_ROOM", "default")
+
+mcp = FastMCP("media")
+
+
+def _media_get(path: str, params: dict | None = None) -> dict:
+    resp = httpx.get(f"{MEDIA_SERVER_URL}{path}", params=params, timeout=10)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def _media_post(path: str, body: dict) -> dict:
+    resp = httpx.post(f"{MEDIA_SERVER_URL}{path}", json=body, timeout=30)
+    if resp.status_code >= 400:
+        try:
+            detail = resp.json().get("error", resp.text)
+        except Exception:  # noqa: BLE001
+            detail = resp.text
+        return {"error": f"{resp.status_code}: {detail}"}
+    return resp.json()
+
+
+def _control(action: str, **extra) -> str:
+    data = _media_post("/media/api/control", {"room": MEDIA_ROOM, "action": action, **extra})
+    if "error" in data:
+        return f"Could not {action} the music: {data['error']}"
+    track = data.get("track") or {}
+    desired = data.get("desired", "")
+    name = track.get("title") or "nothing"
+    return f"OK — {action} applied. Loaded track: {name} (player state: {desired})."
+
+
+def _fmt_duration(seconds) -> str:
+    try:
+        seconds = int(seconds or 0)
+    except (TypeError, ValueError):
+        return ""
+    if seconds <= 0:
+        return ""
+    minutes, secs = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}"
+
+
+@mcp.tool()
+def search_music(query: str, limit: int = 5) -> str:
+    """Search YouTube Music for songs. Call this before playing anything the
+    user describes by name, artist or lyrics. Returns JSON results with
+    video_id, title, channel, duration and in_library flags. Pick the result
+    that clearly matches what the user asked for; if two or more could be
+    right, use offer_song_choices instead of guessing."""
+    data = _media_post("/media/api/search", {"query": query, "limit": max(1, min(limit, 10))})
+    if "error" in data:
+        return f"Search failed: {data['error']}"
+    results = data.get("results", [])
+    if not results:
+        return f"No YouTube Music results for '{query}'."
+    return json.dumps(results, ensure_ascii=False)
+
+
+@mcp.tool()
+def play_song(video_id: str, title: str = "", channel: str = "") -> str:
+    """Play a song in the user's web player. Pass the video_id from
+    search_music (or an in_library track_id). Downloads and transcodes first
+    if needed; playback starts automatically once ready. After calling this,
+    tell the user what is playing in one short sentence and stop talking —
+    the player takes over and the conversation pauses until they return."""
+    data = _media_post("/media/api/play", {
+        "room": MEDIA_ROOM, "video_id": video_id, "title": title, "channel": channel,
+    })
+    if "error" in data:
+        return f"Could not start playback: {data['error']}"
+    track = data.get("track") or {}
+    status = track.get("status")
+    name = track.get("title") or title or video_id
+    if status == "downloading":
+        return (
+            f"'{name}' is downloading and will start playing automatically when "
+            "ready (usually under a minute). Say it's on the way, then hand over "
+            "to the player."
+        )
+    return f"Playing '{name}' now. Hand over to the player with a short sign-off."
+
+
+@mcp.tool()
+def offer_song_choices(choices: list[dict]) -> str:
+    """Offer the user 2-4 song options when a search is ambiguous. Pass a list
+    like [{"video_id": "...", "title": "...", "channel": "...", "duration": 213}].
+    The choices appear as clickable cards in the music panel; the user can
+    also just say which one they want. Briefly name the options aloud, then
+    wait for their pick."""
+    if not choices:
+        return "No choices provided."
+    data = _media_post("/media/api/choices", {"room": MEDIA_ROOM, "choices": choices})
+    if "error" in data:
+        return f"Could not present choices: {data['error']}"
+    names = [
+        f"{c.get('title')}" + (f" by {c['channel']}" if c.get("channel") else "")
+        for c in choices
+    ]
+    return "Choices shown in the player: " + "; ".join(names) + ". Ask the user to pick one."
+
+
+@mcp.tool()
+def pause_music() -> str:
+    """Pause the currently playing music. The user stays in the conversation
+    afterwards."""
+    return _control("pause")
+
+
+@mcp.tool()
+def resume_music() -> str:
+    """Resume the loaded music. Warn the user you'll stop listening while it
+    plays — they'll need to come back to you to talk again."""
+    return _control("resume")
+
+
+@mcp.tool()
+def stop_music() -> str:
+    """Stop playback completely and clear the loaded track."""
+    return _control("stop")
+
+
+@mcp.tool()
+def skip_song() -> str:
+    """Skip to the next track in the library (most recently added first).
+    If the library has nothing else, playback stops."""
+    return _control("skip")
+
+
+@mcp.tool()
+def seek_music(delta_seconds: int) -> str:
+    """Skip forward or backward in the current track. Positive delta_seconds
+    jumps ahead, negative goes back (e.g. -30 rewinds 30 seconds, 20 skips
+    20 seconds forward)."""
+    try:
+        reported = (_media_get("/media/api/state", {"room": MEDIA_ROOM}).get("reported") or {})
+        position = float(reported.get("position") or 0.0)
+    except Exception:  # noqa: BLE001
+        position = 0.0
+    target = max(0.0, position + float(delta_seconds))
+    return _control("seek", position=target)
+
+
+@mcp.tool()
+def now_playing() -> str:
+    """Return what is currently loaded/playing in the user's web player and
+    its playback position. Use when the user asks what's playing or what this
+    song is."""
+    try:
+        st = _media_get("/media/api/state", {"room": MEDIA_ROOM})
+    except Exception as exc:  # noqa: BLE001
+        return f"Media server unreachable: {exc}"
+    track = st.get("track") or {}
+    if not track:
+        return "Nothing is loaded in the player right now."
+    reported = st.get("reported") or {}
+    state = reported.get("state") or st.get("desired") or "unknown"
+    position = _fmt_duration(reported.get("position"))
+    duration = _fmt_duration(track.get("duration"))
+    bits = [f"'{track.get('title')}'"]
+    if track.get("channel"):
+        bits.append(f"by {track['channel']}")
+    if state == "playing" and position:
+        bits.append(f"at {position}" + (f" of {duration}" if duration else ""))
+    else:
+        bits.append(f"(playback state: {state})")
+    return "Currently loaded: " + " ".join(bits) + "."
+
+
+@mcp.tool()
+def list_library() -> str:
+    """List the permanent music library: downloaded YouTube Music tracks and
+    added live streams, ready to play instantly. Use when the user asks what
+    music is available or wants something from their collection."""
+    data = _media_get("/media/api/library")
+    tracks = data.get("tracks", [])
+    if not tracks:
+        return "The library is empty — search for something with search_music first."
+    lines = []
+    for t in tracks[:25]:
+        status = "" if t.get("status") == "ready" else f" ({t.get('status')})"
+        dur = _fmt_duration(t.get("duration"))
+        dur = f" [{dur}]" if dur else ""
+        channel = f" — {t['channel']}" if t.get("channel") else ""
+        lines.append(f"{t['title']}{channel}{dur}{status}")
+    return f"{len(tracks)} tracks in the library:\n" + "\n".join(lines)
+
+
+@mcp.tool()
+def add_stream(name: str, url: str) -> str:
+    """Add a live audio stream URL (internet radio station, direct mp3/aac
+    stream) to the permanent library. It then shows up in the web player's
+    library and can be played by name. Pass a short friendly name."""
+    if not url.lower().startswith(("http://", "https://")):
+        return "That does not look like an http(s) stream URL."
+    data = _media_post("/media/api/streams", {"name": name, "url": url})
+    if "error" in data:
+        return f"Could not add the stream: {data['error']}"
+    return f"Added '{name}' to the library as a live stream. You can play it on request."
+
+
+if __name__ == "__main__":
+    mcp.run(transport="stdio")

agent/pyproject.toml

@@ -9,10 +9,14 @@ description = "Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit
 requires-python = ">=3.10,<3.15"
 
 dependencies = [
+    "fastapi>=0.141.1",
+    "httpx>=0.28.1",
     "livekit-agents[mcp]~=1.7",
     "livekit-plugins-azure~=1.7",
     "livekit-plugins-openai~=1.7",
     "python-dotenv",
+    "uvicorn>=0.52.4",
+    "yt-dlp>=2026.8.19",
 ]
 
 [tool.setuptools.packages.find]

agent/uv.lock

@@ -671,6 +671,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
 ]
 
+[[package]]
+name = "fastapi"
+version = "0.141.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "annotated-doc" },
+    { name = "pydantic" },
+    { name = "starlette" },
+    { name = "typing-extensions" },
+    { name = "typing-inspection" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" },
+]
+
 [[package]]
 name = "frozenlist"
 version = "1.8.0"
@@ -2744,18 +2760,26 @@ name = "voice-agent"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "fastapi" },
+    { name = "httpx" },
     { name = "livekit-agents", extra = ["mcp"] },
     { name = "livekit-plugins-azure" },
     { name = "livekit-plugins-openai" },
     { name = "python-dotenv" },
+    { name = "uvicorn" },
+    { name = "yt-dlp" },
 ]
 
 [package.metadata]
 requires-dist = [
+    { name = "fastapi", specifier = ">=0.141.1" },
+    { name = "httpx", specifier = ">=0.28.1" },
     { name = "livekit-agents", extras = ["mcp"], specifier = "~=1.7" },
     { name = "livekit-plugins-azure", specifier = "~=1.7" },
     { name = "livekit-plugins-openai", specifier = "~=1.7" },
     { name = "python-dotenv" },
+    { name = "uvicorn", specifier = ">=0.52.4" },
+    { name = "yt-dlp", specifier = ">=2026.8.19" },
 ]
 
 [[package]]
@@ -3038,3 +3062,12 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" },
     { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" },
 ]
+
+[[package]]
+name = "yt-dlp"
+version = "2026.8.19"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1e/e0/832fa4ca334b766a06933a196066edc3dba37cdb6f14cd98d59bcc69a4b4/yt_dlp-2026.8.19.tar.gz", hash = "sha256:9e213e48cea35c66b378e4447903f118f6392a5fa380a2b6d7070ec86f4e0af1", size = 3052025, upload-time = "2026-08-19T23:48:59.291Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/69/b2/8cd1613f56eed7ceb64fbd4df3f1c01246bfb098e6f398228bafda22b80b/yt_dlp-2026.8.19-py3-none-any.whl", hash = "sha256:1d57897e94c6665a0a6f9bc54b34e584284e32c034ffab3a7df25d8f7b24eedf", size = 3185533, upload-time = "2026-08-19T23:48:56.925Z" },
+]

agent/voice_agent.egg-info/PKG-INFO

@@ -3,7 +3,11 @@ Name: voice-agent
 Version: 0.1.0
 Summary: Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit Agents
 Requires-Python: <3.15,>=3.10
+Requires-Dist: fastapi>=0.141.1
+Requires-Dist: httpx>=0.28.1
 Requires-Dist: livekit-agents[mcp]~=1.7
 Requires-Dist: livekit-plugins-azure~=1.7
 Requires-Dist: livekit-plugins-openai~=1.7
 Requires-Dist: python-dotenv
+Requires-Dist: uvicorn>=0.52.4
+Requires-Dist: yt-dlp>=2026.8.19

agent/voice_agent.egg-info/requires.txt

@@ -1,4 +1,8 @@
+fastapi>=0.141.1
+httpx>=0.28.1
 livekit-agents[mcp]~=1.7
 livekit-plugins-azure~=1.7
 livekit-plugins-openai~=1.7
 python-dotenv
+uvicorn>=0.52.4
+yt-dlp>=2026.8.19