From 04c4a535948a8fe06124199ecee184e57077b90a Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sat, 29 Aug 2026 13:03:12 -0400 Subject: [PATCH] feat: media player API, web player panel, and Hope music MCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - web/media_server.py: FastAPI service owning the permanent library (library.json + ./media volume), yt-dlp YouTube Music search, queued download jobs with mp3 transcode (ffmpeg), per-room player desired state, and an HTTPS proxy for external live-stream URLs. - agent/media_mcp.py: Hope tools — search_music, play_song, offer_song_choices, pause/resume/stop/skip/seek, now_playing, list_library, add_stream. - agent/agent.py: media toolset wiring + conversation rule (music playing disables the agent mic; re-engaging pauses music). - web UI: slide-out Music panel (search, suggestions, library, live stream add, play/pause, ±10s, seek, volume) with a reconciling poller and media_state data-channel reports. - token server returns the minted room name; nginx serves /media/files as static (native Range) and proxies /media/api to the media server. - supervisord runs media-server; compose mounts ./media and adds ffmpeg to the image. --- .env.example | 13 + Dockerfile | 12 +- agent/agent.py | 66 ++- agent/media_mcp.py | 229 +++++++++ agent/pyproject.toml | 4 + agent/uv.lock | 33 ++ agent/voice_agent.egg-info/PKG-INFO | 4 + agent/voice_agent.egg-info/requires.txt | 4 + docker-compose.yml | 6 +- nginx.conf | 20 + supervisord.conf | 15 + web/app.js | 516 +++++++++++++++++++- web/index.html | 84 ++++ web/media_server.py | 619 ++++++++++++++++++++++++ web/style.css | 260 +++++++++- web/token_server.py | 3 + 16 files changed, 1876 insertions(+), 12 deletions(-) create mode 100644 agent/media_mcp.py create mode 100644 web/media_server.py diff --git a/.env.example b/.env.example index 2879ede..877bf53 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,19 @@ FIRECRAWL_BASE=http://192.168.86.2:3002 # Per-user memory profiles and Bearer tokens live in config/users.json. VOICE_DEFAULT_USER=shane +# ── Media player (music library + YouTube Music downloads) ───────────────── +# The permanent library lives in ./media (bind-mounted to /data/media): +# media/files/ transcoded mp3 tracks +# media/thumbs/ cover art +# media/jobs/ download job status +# MEDIA_SERVER_PORT=8092 +# +# Optional: export cookies.txt from a browser logged into YouTube and drop it +# in ./media/cookies.txt to avoid "confirm you're not a bot" download blocks. +# Anonymous access works from most residential IPs; cookies are the reliable +# path. +# YTDLP_COOKIES=/data/media/cookies.txt + # Extra MCP servers (optional). JSON list of {url, transport} objects. # transport: "sse" | "streamable_http" | omit for auto-detect # EXTRA_MCP_SERVERS=[{"url":"http://192.168.86.2:3001/mcp","transport":"streamable_http"}] diff --git a/Dockerfile b/Dockerfile index b4c676d..16beb48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,10 @@ ENV PYTHONUNBUFFERED=1 # Fix SSL cert path for the slim base image (ca-certificates installs to /etc/ssl/certs) ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -# Install LiveKit server binary + supervisord + nginx +# Install LiveKit server binary + supervisord + nginx + ffmpeg (yt-dlp audio +# extraction/transcoding for the media player) RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates supervisor nginx libasound2 iproute2 \ + curl ca-certificates supervisor nginx libasound2 iproute2 ffmpeg \ && rm -rf /var/lib/apt/lists/* # Download LiveKit server (latest stable) @@ -48,8 +49,9 @@ COPY agent/task_registry.py /opt/voice-agent/task_registry.py COPY agent/task_router.py /opt/voice-agent/task_router.py COPY agent/task_worker.py /opt/voice-agent/task_worker.py COPY agent/dispatch_mcp.py /opt/voice-agent/dispatch_mcp.py +COPY agent/media_mcp.py /opt/voice-agent/media_mcp.py -# Copy web frontend + token endpoint +# Copy web frontend + token endpoint + media server COPY web/index.html /var/www/voice/ COPY web/app.js /var/www/voice/ COPY web/style.css /var/www/voice/ @@ -57,6 +59,10 @@ COPY web/livekit-client.umd.js /var/www/voice/ COPY web/manifest.json /var/www/voice/ COPY web/favicon.svg /var/www/voice/ COPY web/token_server.py /opt/voice/token_server.py +COPY web/media_server.py /opt/voice/media_server.py + +# Permanent media library mount point (bind-mounted from ./media) +RUN mkdir -p /data/media/files /data/media/thumbs /data/media/jobs # Config files COPY livekit.yaml /etc/livekit.yaml diff --git a/agent/agent.py b/agent/agent.py index f8b66f2..3f68f55 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -196,6 +196,23 @@ SYSTEM_PROMPT = textwrap.dedent("""\ you receive it, share the findings naturally in your conversational style. You can have multiple tasks running at once. + # Music (media player in the web UI) + You can play music on the user's web player. For any music request, call + search_music first, then: + - One clearly-right result → play_song with its video_id. Announce it in + one short sentence, then stop talking — playback pauses the + conversation until the user comes back to you. + - Ambiguous results → offer_song_choices with the top 2-3 options, name + them briefly, and let the user pick by voice or in the player panel. + Never start playback on a guess. + The library is permanent: list_library shows what is already downloaded, + and those tracks play instantly without searching. If the user names a + library track exactly, just play it. Control playback with pause_music, + resume_music, skip_song and seek_music; check what's on with now_playing. + add_stream registers a live radio or direct audio URL as a library track. + While music is playing you cannot hear the user — they will come back to + you when they want to talk again, and the music pauses automatically. + # Skills You have a skill library of learned procedures. Use skill_recall when you need to perform a task you've done before — it will give you the steps. When you @@ -624,6 +641,27 @@ def build_mcp_toolsets(user_profile: dict, task_dir: str) -> list[mcp.MCPToolset ) logger.info("Dispatch MCP toolset enabled (%s)", task_dir) + # Media tools talk to the local media server; the room name scopes player + # state so concurrent sessions don't fight over one virtual player. + media_server_url = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092") + media_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media_mcp.py") + toolsets.append( + mcp.MCPToolset( + id="media", + mcp_server=mcp.MCPServerStdio( + command=python_bin, + args=[media_mcp_script], + env={ + **os.environ, + "MEDIA_SERVER_URL": media_server_url, + "MEDIA_ROOM": os.path.basename(task_dir.rstrip("/")) or "default", + }, + client_session_timeout_seconds=60, + ), + ) + ) + logger.info("Media MCP toolset enabled (%s)", media_server_url) + extra = os.environ.get("EXTRA_MCP_SERVERS", "") if extra: try: @@ -740,9 +778,22 @@ async def handle_job(ctx: JobContext) -> None: # Listen for conversation controls (voice selection and voice/text mode). # Defined here so it captures this job's session via closure. + # Music and conversation are mutually exclusive by design: while a track + # plays the agent's ears are off; re-engaging the conversation pauses the + # music, and ending the conversation lets the user resume it. + session_mode = {"value": "voice"} # UI voice/text switch + media_playback = {"value": "stopped"} # UI-reported player state + + def _apply_listening() -> None: + listening = ( + session_mode["value"] == "voice" + and media_playback["value"] != "playing" + ) + session.input.set_audio_enabled(listening) + def _on_room_data(packet) -> None: topic = getattr(packet, "topic", None) - if topic and topic not in ("voice-control", "conversation-control"): + if topic and topic not in ("voice-control", "conversation-control", "media-control"): return try: msg = json.loads(packet.data.decode("utf-8")) @@ -755,11 +806,22 @@ async def handle_job(ctx: JobContext) -> None: return voice_enabled = mode == "voice" logger.info("Switching conversation mode to %s", mode) + session_mode["value"] = mode if not voice_enabled: # interrupt() returns an already-scheduled Future, not a coroutine. session.interrupt() - session.input.set_audio_enabled(voice_enabled) session.output.set_audio_enabled(voice_enabled) + _apply_listening() + return + + if msg.get("type") == "media_state": + state = msg.get("state") + if state not in ("playing", "paused", "stopped"): + return + logger.info("Media playback -> %s (agent listening: %s)", + state, session_mode["value"] == "voice" and state != "playing") + media_playback["value"] = state + _apply_listening() return if msg.get("type") == "set_voice": diff --git a/agent/media_mcp.py b/agent/media_mcp.py new file mode 100644 index 0000000..2bd7201 --- /dev/null +++ b/agent/media_mcp.py @@ -0,0 +1,229 @@ +"""Media MCP server — lets Hope find and control music by voice. + +Tools talk to the local media server (web/media_server.py) over HTTP, which +owns the library, YouTube Music search/download jobs and per-room player +state. The browser is the actual player: it reconciles against the media +server and reports playback back, so voice controls and the web panel stay +in sync. + +Per-session environment (set by the agent when spawning this MCP): +- MEDIA_SERVER_URL e.g. http://127.0.0.1:8092 +- MEDIA_ROOM LiveKit room name this session controls + +The "stop listening while music plays" rule lives in the agent itself: the +UI reports media_state over the data channel and the agent disables its mic +while a track is playing. +""" +from __future__ import annotations + +import json +import os + +import httpx +from mcp.server.fastmcp import FastMCP + +MEDIA_SERVER_URL = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092").rstrip("/") +MEDIA_ROOM = os.environ.get("MEDIA_ROOM", "default") + +mcp = FastMCP("media") + + +def _media_get(path: str, params: dict | None = None) -> dict: + resp = httpx.get(f"{MEDIA_SERVER_URL}{path}", params=params, timeout=10) + resp.raise_for_status() + return resp.json() + + +def _media_post(path: str, body: dict) -> dict: + resp = httpx.post(f"{MEDIA_SERVER_URL}{path}", json=body, timeout=30) + if resp.status_code >= 400: + try: + detail = resp.json().get("error", resp.text) + except Exception: # noqa: BLE001 + detail = resp.text + return {"error": f"{resp.status_code}: {detail}"} + return resp.json() + + +def _control(action: str, **extra) -> str: + data = _media_post("/media/api/control", {"room": MEDIA_ROOM, "action": action, **extra}) + if "error" in data: + return f"Could not {action} the music: {data['error']}" + track = data.get("track") or {} + desired = data.get("desired", "") + name = track.get("title") or "nothing" + return f"OK — {action} applied. Loaded track: {name} (player state: {desired})." + + +def _fmt_duration(seconds) -> str: + try: + seconds = int(seconds or 0) + except (TypeError, ValueError): + return "" + if seconds <= 0: + return "" + minutes, secs = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +@mcp.tool() +def search_music(query: str, limit: int = 5) -> str: + """Search YouTube Music for songs. Call this before playing anything the + user describes by name, artist or lyrics. Returns JSON results with + video_id, title, channel, duration and in_library flags. Pick the result + that clearly matches what the user asked for; if two or more could be + right, use offer_song_choices instead of guessing.""" + data = _media_post("/media/api/search", {"query": query, "limit": max(1, min(limit, 10))}) + if "error" in data: + return f"Search failed: {data['error']}" + results = data.get("results", []) + if not results: + return f"No YouTube Music results for '{query}'." + return json.dumps(results, ensure_ascii=False) + + +@mcp.tool() +def play_song(video_id: str, title: str = "", channel: str = "") -> str: + """Play a song in the user's web player. Pass the video_id from + search_music (or an in_library track_id). Downloads and transcodes first + if needed; playback starts automatically once ready. After calling this, + tell the user what is playing in one short sentence and stop talking — + the player takes over and the conversation pauses until they return.""" + data = _media_post("/media/api/play", { + "room": MEDIA_ROOM, "video_id": video_id, "title": title, "channel": channel, + }) + if "error" in data: + return f"Could not start playback: {data['error']}" + track = data.get("track") or {} + status = track.get("status") + name = track.get("title") or title or video_id + if status == "downloading": + return ( + f"'{name}' is downloading and will start playing automatically when " + "ready (usually under a minute). Say it's on the way, then hand over " + "to the player." + ) + return f"Playing '{name}' now. Hand over to the player with a short sign-off." + + +@mcp.tool() +def offer_song_choices(choices: list[dict]) -> str: + """Offer the user 2-4 song options when a search is ambiguous. Pass a list + like [{"video_id": "...", "title": "...", "channel": "...", "duration": 213}]. + The choices appear as clickable cards in the music panel; the user can + also just say which one they want. Briefly name the options aloud, then + wait for their pick.""" + if not choices: + return "No choices provided." + data = _media_post("/media/api/choices", {"room": MEDIA_ROOM, "choices": choices}) + if "error" in data: + return f"Could not present choices: {data['error']}" + names = [ + f"{c.get('title')}" + (f" by {c['channel']}" if c.get("channel") else "") + for c in choices + ] + return "Choices shown in the player: " + "; ".join(names) + ". Ask the user to pick one." + + +@mcp.tool() +def pause_music() -> str: + """Pause the currently playing music. The user stays in the conversation + afterwards.""" + return _control("pause") + + +@mcp.tool() +def resume_music() -> str: + """Resume the loaded music. Warn the user you'll stop listening while it + plays — they'll need to come back to you to talk again.""" + return _control("resume") + + +@mcp.tool() +def stop_music() -> str: + """Stop playback completely and clear the loaded track.""" + return _control("stop") + + +@mcp.tool() +def skip_song() -> str: + """Skip to the next track in the library (most recently added first). + If the library has nothing else, playback stops.""" + return _control("skip") + + +@mcp.tool() +def seek_music(delta_seconds: int) -> str: + """Skip forward or backward in the current track. Positive delta_seconds + jumps ahead, negative goes back (e.g. -30 rewinds 30 seconds, 20 skips + 20 seconds forward).""" + try: + reported = (_media_get("/media/api/state", {"room": MEDIA_ROOM}).get("reported") or {}) + position = float(reported.get("position") or 0.0) + except Exception: # noqa: BLE001 + position = 0.0 + target = max(0.0, position + float(delta_seconds)) + return _control("seek", position=target) + + +@mcp.tool() +def now_playing() -> str: + """Return what is currently loaded/playing in the user's web player and + its playback position. Use when the user asks what's playing or what this + song is.""" + try: + st = _media_get("/media/api/state", {"room": MEDIA_ROOM}) + except Exception as exc: # noqa: BLE001 + return f"Media server unreachable: {exc}" + track = st.get("track") or {} + if not track: + return "Nothing is loaded in the player right now." + reported = st.get("reported") or {} + state = reported.get("state") or st.get("desired") or "unknown" + position = _fmt_duration(reported.get("position")) + duration = _fmt_duration(track.get("duration")) + bits = [f"'{track.get('title')}'"] + if track.get("channel"): + bits.append(f"by {track['channel']}") + if state == "playing" and position: + bits.append(f"at {position}" + (f" of {duration}" if duration else "")) + else: + bits.append(f"(playback state: {state})") + return "Currently loaded: " + " ".join(bits) + "." + + +@mcp.tool() +def list_library() -> str: + """List the permanent music library: downloaded YouTube Music tracks and + added live streams, ready to play instantly. Use when the user asks what + music is available or wants something from their collection.""" + data = _media_get("/media/api/library") + tracks = data.get("tracks", []) + if not tracks: + return "The library is empty — search for something with search_music first." + lines = [] + for t in tracks[:25]: + status = "" if t.get("status") == "ready" else f" ({t.get('status')})" + dur = _fmt_duration(t.get("duration")) + dur = f" [{dur}]" if dur else "" + channel = f" — {t['channel']}" if t.get("channel") else "" + lines.append(f"{t['title']}{channel}{dur}{status}") + return f"{len(tracks)} tracks in the library:\n" + "\n".join(lines) + + +@mcp.tool() +def add_stream(name: str, url: str) -> str: + """Add a live audio stream URL (internet radio station, direct mp3/aac + stream) to the permanent library. It then shows up in the web player's + library and can be played by name. Pass a short friendly name.""" + if not url.lower().startswith(("http://", "https://")): + return "That does not look like an http(s) stream URL." + data = _media_post("/media/api/streams", {"name": name, "url": url}) + if "error" in data: + return f"Could not add the stream: {data['error']}" + return f"Added '{name}' to the library as a live stream. You can play it on request." + + +if __name__ == "__main__": + mcp.run(transport="stdio") diff --git a/agent/pyproject.toml b/agent/pyproject.toml index 1b08d03..6dbb2d5 100644 --- a/agent/pyproject.toml +++ b/agent/pyproject.toml @@ -9,10 +9,14 @@ description = "Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit requires-python = ">=3.10,<3.15" dependencies = [ + "fastapi>=0.141.1", + "httpx>=0.28.1", "livekit-agents[mcp]~=1.7", "livekit-plugins-azure~=1.7", "livekit-plugins-openai~=1.7", "python-dotenv", + "uvicorn>=0.52.4", + "yt-dlp>=2026.8.19", ] [tool.setuptools.packages.find] diff --git a/agent/uv.lock b/agent/uv.lock index 01932ea..dd5bd2f 100644 --- a/agent/uv.lock +++ b/agent/uv.lock @@ -671,6 +671,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -2744,18 +2760,26 @@ name = "voice-agent" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "fastapi" }, + { name = "httpx" }, { name = "livekit-agents", extra = ["mcp"] }, { name = "livekit-plugins-azure" }, { name = "livekit-plugins-openai" }, { name = "python-dotenv" }, + { name = "uvicorn" }, + { name = "yt-dlp" }, ] [package.metadata] requires-dist = [ + { name = "fastapi", specifier = ">=0.141.1" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "livekit-agents", extras = ["mcp"], specifier = "~=1.7" }, { name = "livekit-plugins-azure", specifier = "~=1.7" }, { name = "livekit-plugins-openai", specifier = "~=1.7" }, { name = "python-dotenv" }, + { name = "uvicorn", specifier = ">=0.52.4" }, + { name = "yt-dlp", specifier = ">=2026.8.19" }, ] [[package]] @@ -3038,3 +3062,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" }, { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, ] + +[[package]] +name = "yt-dlp" +version = "2026.8.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e0/832fa4ca334b766a06933a196066edc3dba37cdb6f14cd98d59bcc69a4b4/yt_dlp-2026.8.19.tar.gz", hash = "sha256:9e213e48cea35c66b378e4447903f118f6392a5fa380a2b6d7070ec86f4e0af1", size = 3052025, upload-time = "2026-08-19T23:48:59.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/8cd1613f56eed7ceb64fbd4df3f1c01246bfb098e6f398228bafda22b80b/yt_dlp-2026.8.19-py3-none-any.whl", hash = "sha256:1d57897e94c6665a0a6f9bc54b34e584284e32c034ffab3a7df25d8f7b24eedf", size = 3185533, upload-time = "2026-08-19T23:48:56.925Z" }, +] diff --git a/agent/voice_agent.egg-info/PKG-INFO b/agent/voice_agent.egg-info/PKG-INFO index 90994b1..01f7c06 100644 --- a/agent/voice_agent.egg-info/PKG-INFO +++ b/agent/voice_agent.egg-info/PKG-INFO @@ -3,7 +3,11 @@ Name: voice-agent Version: 0.1.0 Summary: Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit Agents Requires-Python: <3.15,>=3.10 +Requires-Dist: fastapi>=0.141.1 +Requires-Dist: httpx>=0.28.1 Requires-Dist: livekit-agents[mcp]~=1.7 Requires-Dist: livekit-plugins-azure~=1.7 Requires-Dist: livekit-plugins-openai~=1.7 Requires-Dist: python-dotenv +Requires-Dist: uvicorn>=0.52.4 +Requires-Dist: yt-dlp>=2026.8.19 diff --git a/agent/voice_agent.egg-info/requires.txt b/agent/voice_agent.egg-info/requires.txt index 6f2c05e..72abc97 100644 --- a/agent/voice_agent.egg-info/requires.txt +++ b/agent/voice_agent.egg-info/requires.txt @@ -1,4 +1,8 @@ +fastapi>=0.141.1 +httpx>=0.28.1 livekit-agents[mcp]~=1.7 livekit-plugins-azure~=1.7 livekit-plugins-openai~=1.7 python-dotenv +uvicorn>=0.52.4 +yt-dlp>=2026.8.19 diff --git a/docker-compose.yml b/docker-compose.yml index 8e704ec..296cd6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,12 +18,16 @@ services: WEB_PORT: "${WEB_PORT:-9178}" VOICE_USERS_FILE: "/etc/hope/users.json" VOICE_DEFAULT_USER: "${VOICE_DEFAULT_USER:-shane}" + MEDIA_DIR: "/data/media" + MEDIA_SERVER_PORT: "${MEDIA_SERVER_PORT:-8092}" + YTDLP_COOKIES: "${YTDLP_COOKIES:-}" volumes: - ./livekit.yaml:/etc/livekit.yaml:ro - ./certs:/etc/voice/certs - ./skills:/skills + - ./media:/data/media healthcheck: - test: ["CMD-SHELL", "curl -sk https://localhost:$${WEB_PORT:-9178}/ -o /dev/null && curl -s http://localhost:7880/ -o /dev/null"] + test: ["CMD-SHELL", "curl -sk https://localhost:$${WEB_PORT:-9178}/ -o /dev/null && curl -s http://localhost:7880/ -o /dev/null && curl -s http://localhost:$${MEDIA_SERVER_PORT:-8092}/media/api/healthz -o /dev/null"] interval: 15s timeout: 5s retries: 3 diff --git a/nginx.conf b/nginx.conf index fdd9d2c..cdfe81e 100644 --- a/nginx.conf +++ b/nginx.conf @@ -23,6 +23,26 @@ server { proxy_send_timeout 3600s; } + # Media server API (library, search, player state) and the proxy for + # external live-stream URLs (http:// sources would be blocked as mixed + # content in the browser, so they are re-served over HTTPS from here). + location /media/api/ { + proxy_pass http://127.0.0.1:8092/media/api/; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + + # Downloaded tracks are plain static files — nginx serves them with + # native HTTP Range support, which gives the web player seeking for free. + location /media/files/ { + alias /data/media/files/; + } + + location /media/thumbs/ { + alias /data/media/thumbs/; + } + location / { try_files $uri $uri/ =404; } diff --git a/supervisord.conf b/supervisord.conf index 5a89ff0..4336f14 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -56,3 +56,18 @@ environment= LIVEKIT_API_SECRET="%(ENV_LIVEKIT_API_SECRET)s", VOICE_USERS_FILE="%(ENV_VOICE_USERS_FILE)s", VOICE_DEFAULT_USER="%(ENV_VOICE_DEFAULT_USER)s" + +[program:media-server] +; Runs as root because the /data/media bind mount is a host (Unraid) share +; whose uid does not match the container's voiceuser. +command=/opt/voice-agent/.venv/bin/python /opt/voice/media_server.py +user=root +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true +environment= + MEDIA_DIR="%(ENV_MEDIA_DIR)s", + MEDIA_SERVER_PORT="%(ENV_MEDIA_SERVER_PORT)s", + YTDLP_COOKIES="%(ENV_YTDLP_COOKIES)s" diff --git a/web/app.js b/web/app.js index c6bff9e..591f016 100644 --- a/web/app.js +++ b/web/app.js @@ -693,6 +693,7 @@ document.addEventListener("keydown", (event) => { if (event.key !== "Escape") return; if (settingsSheet.classList.contains("open")) closeSheet(); if (tasksPanel.classList.contains("open")) setTasksOpen(false); + if (mediaPanel.classList.contains("open")) setMediaOpen(false); }); // ── Wake lock ─────────────────────────────────────────────────────────────── async function acquireWakeLock() { @@ -816,6 +817,8 @@ function createRoom(token) { stopBtn.disabled = true; agentParticipant = null; agentReady = false; + // Leaving the conversation hands the room back to the music player. + resumePlaybackAfterTalk(); }); return newRoom.connect(LIVEKIT_URL, token, { @@ -873,8 +876,9 @@ async function connectConversation() { await acquireWakeLock(); // Get a signed access token from the token endpoint. - const token = await fetchToken(ROOM_NAME); - room = await createRoom(token); + const tokenData = await fetchToken(ROOM_NAME); + await enterMediaScope(tokenData.room); + room = await createRoom(tokenData.token); // The agent is dispatched when we join. If it misses the dispatch // (for example during a server restart), reconnect once. @@ -886,8 +890,9 @@ async function connectConversation() { agentParticipant = null; agentReady = false; await stale.disconnect(); - const freshToken = await fetchToken(ROOM_NAME); - room = await createRoom(freshToken); + const freshTokenData = await fetchToken(ROOM_NAME); + await enterMediaScope(freshTokenData.room); + room = await createRoom(freshTokenData.token); if (!(await waitForAgent(15000))) { throw new Error("Assistant did not join — reload the page and try again"); } @@ -999,6 +1004,8 @@ stopBtn.addEventListener("click", () => { muteBtn.disabled = true; micMuted = false; updateMuteIcon(); + // Ending the conversation resumes music that was paused for the talk. + resumePlaybackAfterTalk(); }); // ── Mic mute toggle ───────────────────────────────────────────────────────── @@ -1030,7 +1037,7 @@ async function fetchToken(roomName) { }); if (!resp.ok) throw new Error(`Token endpoint returned ${resp.status}`); const data = await resp.json(); - return data.token; + return { token: data.token, room: data.room || null }; } // ── Helper: send voice selection to the agent via data channel ───────────── @@ -1064,3 +1071,502 @@ function sendModelMessage(model, targetRoom) { reliable: true, }); } + +// ── Media player ──────────────────────────────────────────────────────────── +// The browser