From ce8fd1e04695b369e50553ff1fffe62cb82903c0 Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sat, 29 Aug 2026 16:01:26 -0400 Subject: [PATCH] =?UTF-8?q?fix:=20move=20media=20server=20to=208093=20?= =?UTF-8?q?=E2=80=94=20YOURLS/Apache=20owns=208092=20on=20xNAS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Unraid deployment's /media/api requests were answered by an Apache/YOURLS stack squatting on 8092, surfacing as UI 503s; the CI healthcheck (curl without -f) counted that 503 as healthy, so run #15 deployed green against a broken media server. - media server port 8092 -> 8093 everywhere (server, MCP, agent, nginx, compose, CI, .env.example, docs, tests). - healthchecks use curl -f in compose and rebuild.yml so HTTP error statuses fail deployment. - tests default to the Unraid deployment (VOICE_WEB_URL= https://192.168.86.2:9178); token goes through nginx (loopback :8091 is unreachable off-host); bind-mount-coupled tests skip on remote targets. AGENTS.md documents the convention and both failure modes. --- .env.example | 2 +- .gitea/workflows/rebuild.yml | 4 ++-- AGENTS.md | 20 ++++++++++++++++++-- agent/agent.py | 2 +- agent/media_mcp.py | 4 ++-- docker-compose.yml | 4 ++-- nginx.conf | 2 +- tests/conftest.py | 29 +++++++++++++++++++++-------- tests/test_media.py | 21 +++++++++++++++------ tests/test_web_ui.py | 7 +++++-- web/media_server.py | 5 ++++- 11 files changed, 72 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index 877bf53..7e8bcda 100644 --- a/.env.example +++ b/.env.example @@ -34,7 +34,7 @@ VOICE_DEFAULT_USER=shane # media/files/ transcoded mp3 tracks # media/thumbs/ cover art # media/jobs/ download job status -# MEDIA_SERVER_PORT=8092 +# MEDIA_SERVER_PORT=8093 (8092 is taken by YOURLS/Apache on xNAS) # # Optional: export cookies.txt from a browser logged into YouTube and drop it # in ./media/cookies.txt to avoid "confirm you're not a bot" download blocks. diff --git a/.gitea/workflows/rebuild.yml b/.gitea/workflows/rebuild.yml index 65fb730..0856d52 100644 --- a/.gitea/workflows/rebuild.yml +++ b/.gitea/workflows/rebuild.yml @@ -26,7 +26,7 @@ jobs: --name voice \ --network host \ --restart unless-stopped \ - --health-cmd='curl -sk https://localhost:${WEB_PORT:-9178}/ -o /dev/null && curl -s http://localhost:7880/ -o /dev/null && curl -s http://localhost:${MEDIA_SERVER_PORT:-8092}/media/api/healthz -o /dev/null' \ + --health-cmd='curl -sfk https://localhost:${WEB_PORT:-9178}/ -o /dev/null && curl -sf http://localhost:7880/ -o /dev/null && curl -sf http://localhost:${MEDIA_SERVER_PORT:-8093}/media/api/healthz -o /dev/null' \ --health-interval=15s \ --health-timeout=5s \ --health-retries=3 \ @@ -50,7 +50,7 @@ jobs: -e VOICE_USERS_FILE=/etc/hope/users.json \ -e VOICE_DEFAULT_USER=shane \ -e MEDIA_DIR=/data/media \ - -e MEDIA_SERVER_PORT=8092 \ + -e MEDIA_SERVER_PORT=8093 \ -e YTDLP_COOKIES=/data/media/cookies.txt \ hope-voice-api-voice:latest diff --git a/AGENTS.md b/AGENTS.md index f048734..e857907 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ One Docker container runs four processes via supervisord: 1. **LiveKit server** — open-source WebRTC media transport (port 7880 TCP, 7882 UDP muxed media) 2. **Voice agent** — Python LiveKit Agents pipeline: Azure STT → LLM (OpenWebUI) → Azure TTS 3. **Web frontend** — static HTML served by nginx (port 9178 by default) -4. **Media server** — FastAPI music library/player coordinator (port 8092 internal, proxied at /media/api/) +4. **Media server** — FastAPI music library/player coordinator (port 8093 internal, proxied at /media/api/) ``` Browser ──WebRTC──► LiveKit Server ──audio──► Agent @@ -44,7 +44,7 @@ LAN access requires firewall rules: `9178/tcp` (UI + signaling), `7882/udp` | 7880 | TCP | LiveKit HTTP/WS | internal only (proxied via nginx at /livekit/) | | 7881 | TCP | LiveKit RTC media (TCP fallback) | LAN | | 7882 | UDP | LiveKit RTC media (muxed) | LAN | -| 8092 | TCP | Media server (library, player state, stream proxy) | internal only (proxied via nginx at /media/api/) | +| 8093 | TCP | Media server (library, player state, stream proxy) | internal only (proxied via nginx at /media/api/) | | 9178 | TCP | Web frontend (HTTPS) | LAN | ## Configuration @@ -107,10 +107,23 @@ The LLM is instructed to: ## Testing +**Tests run against the Unraid deployment by default** (`https://192.168.86.2:9178`, +via `VOICE_WEB_URL` in tests/conftest.py) — that is the instance users hit +and the one CI (rebuild.yml) deploys on every push to master. The local dev +compose instance is for development only; set `VOICE_WEB_URL=https://localhost:9178` +to target it instead. + ```bash # Verify the container is running docker compose ps +# Run the suite against the Unraid deployment (default) +python -m pytest tests/ -v -m "not slow" + +# Real end-to-end YouTube Music download (network-heavy, gated) +HOPE_RUN_YTDLP_DOWNLOAD=1 python -m pytest tests/test_media.py -m slow +``` + # Check all logs (single service: "voice") docker compose logs -f voice @@ -179,6 +192,9 @@ curl -s "https://eastus.tts.speech.microsoft.com/cognitiveservices/v1" \ - **External stream URLs are proxied, never played directly.** An `http://` radio URL would be blocked as mixed content on the HTTPS UI, so `GET /media/api/stream/{id}` re-serves the upstream over HTTPS with Range passthrough. Downloaded tracks on that endpoint just redirect to `/media/files/`. - **Player state is per-room desired state, reconciled by the browser.** The media server never streams to anything; it records what SHOULD play per LiveKit room and the browser polls and reports back. Track ids for YouTube tracks are the video ids (dedupe by video_id); stream tracks get `s`. - **Anonymous yt-dlp first, cookies when flaky.** Search (`ytsearch`) and downloads work anonymously from a residential IP; if YouTube starts bot-blocking, drop a cookies.txt into `./media/cookies.txt` and set `YTDLP_COOKIES=/data/media/cookies.txt`. +- **Never hard-require container env in supervisord.** `%(ENV_X)s` expansion aborts the whole supervisord config when the var is missing, taking every program down (this silently killed the run-151 deploy). Programs with safe in-code defaults (media server) must inherit the container env instead of listing it. +- **Healthchecks must use `curl -f`.** Without it, any HTTP response — including a 503 from a squatter — counts as healthy. Run #15 deployed "healthy" against a broken media port this way. +- **Port 8092 is taken on xNAS.** An Apache/YOURLS stack occupies 8092 on the Unraid host; the media server uses 8093. Before picking a new internal port, probe the Unraid deployment host, not just the dev box. ## Git diff --git a/agent/agent.py b/agent/agent.py index 70d674c..1ca8ee9 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -688,7 +688,7 @@ def build_mcp_toolsets(user_profile: dict, task_dir: str) -> list[mcp.MCPToolset # Media tools talk to the local media server; the room name scopes player # state so concurrent sessions don't fight over one virtual player. - media_server_url = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092") + media_server_url = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8093") media_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media_mcp.py") toolsets.append( mcp.MCPToolset( diff --git a/agent/media_mcp.py b/agent/media_mcp.py index 2bd7201..7d44cab 100644 --- a/agent/media_mcp.py +++ b/agent/media_mcp.py @@ -7,7 +7,7 @@ server and reports playback back, so voice controls and the web panel stay in sync. Per-session environment (set by the agent when spawning this MCP): -- MEDIA_SERVER_URL e.g. http://127.0.0.1:8092 +- MEDIA_SERVER_URL e.g. http://127.0.0.1:8093 - MEDIA_ROOM LiveKit room name this session controls The "stop listening while music plays" rule lives in the agent itself: the @@ -22,7 +22,7 @@ import os import httpx from mcp.server.fastmcp import FastMCP -MEDIA_SERVER_URL = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092").rstrip("/") +MEDIA_SERVER_URL = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8093").rstrip("/") MEDIA_ROOM = os.environ.get("MEDIA_ROOM", "default") mcp = FastMCP("media") diff --git a/docker-compose.yml b/docker-compose.yml index 18c65a9..a5c55fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: VOICE_USERS_FILE: "/etc/hope/users.json" VOICE_DEFAULT_USER: "${VOICE_DEFAULT_USER:-shane}" MEDIA_DIR: "/data/media" - MEDIA_SERVER_PORT: "${MEDIA_SERVER_PORT:-8092}" + MEDIA_SERVER_PORT: "${MEDIA_SERVER_PORT:-8093}" YTDLP_COOKIES: "${YTDLP_COOKIES:-/data/media/cookies.txt}" volumes: - ./livekit.yaml:/etc/livekit.yaml:ro @@ -27,7 +27,7 @@ services: - ./skills:/skills - ./media:/data/media healthcheck: - test: ["CMD-SHELL", "curl -sk https://localhost:$${WEB_PORT:-9178}/ -o /dev/null && curl -s http://localhost:7880/ -o /dev/null && curl -s http://localhost:$${MEDIA_SERVER_PORT:-8092}/media/api/healthz -o /dev/null"] + test: ["CMD-SHELL", "curl -sfk https://localhost:$${WEB_PORT:-9178}/ -o /dev/null && curl -sf http://localhost:7880/ -o /dev/null && curl -sf http://localhost:$${MEDIA_SERVER_PORT:-8093}/media/api/healthz -o /dev/null"] interval: 15s timeout: 5s retries: 3 diff --git a/nginx.conf b/nginx.conf index cdfe81e..ee2b7a6 100644 --- a/nginx.conf +++ b/nginx.conf @@ -27,7 +27,7 @@ server { # external live-stream URLs (http:// sources would be blocked as mixed # content in the browser, so they are re-served over HTTPS from here). location /media/api/ { - proxy_pass http://127.0.0.1:8092/media/api/; + proxy_pass http://127.0.0.1:8093/media/api/; proxy_buffering off; proxy_read_timeout 3600s; proxy_send_timeout 3600s; diff --git a/tests/conftest.py b/tests/conftest.py index 61c8e8a..3cdaf45 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,23 +1,35 @@ """Shared fixtures for the Hope voice-assistant test suite. -Tests run on the HOST machine against the live "voice" container: -- HTTPS web UI -> https://localhost:9178 (self-signed cert, verify=False) -- Token server -> http://127.0.0.1:8091 (host network mode) +Tests run against the **Unraid deployment** by default — that is the +instance users hit and the one CI (rebuild.yml) keeps up to date: +- HTTPS web UI -> https://192.168.86.2:9178 (self-signed cert, verify=False) +- Token server -> via the same nginx (https://.../token; the loopback + :8091 socket is only reachable on the deployment host itself) - LLM API -> LLM_BASE_URL (default http://192.168.86.2:3005/api) -- In-container -> docker exec voice ... +- In-container -> docker exec voice ... (targets the LOCAL dev container + unless a docker context/DOCKER_HOST for the Unraid host is configured) + +Override VOICE_WEB_URL to test a different instance (e.g. the local dev +compose deployment at https://localhost:9178). """ import json import os import subprocess from pathlib import Path +from urllib.parse import urlparse import httpx import pytest CONTAINER = "voice" -WEB_BASE_URL = os.environ.get("VOICE_WEB_URL", "https://localhost:9178") -TOKEN_URL = os.environ.get("VOICE_TOKEN_URL", "http://127.0.0.1:8091/token") +WEB_BASE_URL = os.environ.get("VOICE_WEB_URL", "https://192.168.86.2:9178").rstrip("/") +TOKEN_URL = os.environ.get( + "VOICE_TOKEN_URL", f"{WEB_BASE_URL}/token" +) +# Filesystem-coupled tests (repo ./media bind mount) only make sense when +# the test host IS the deployment host. +IS_LOCAL_DEPLOYMENT = urlparse(WEB_BASE_URL).hostname in ("localhost", "127.0.0.1") LLM_BASE_URL = ( os.environ.get("LLM_BASE_URL") or os.environ.get("GEMMA_BASE_URL") @@ -91,8 +103,9 @@ def client(base_url): @pytest.fixture(scope="session") def token() -> dict: - """Fetch a fresh LiveKit token from the token endpoint (direct, no proxy).""" - resp = httpx.post(TOKEN_URL, json={}, timeout=10) + """Fetch a fresh LiveKit token (via nginx by default; the loopback + :8091 socket is only reachable on the deployment host itself).""" + resp = httpx.post(TOKEN_URL, json={}, timeout=10, verify=False) resp.raise_for_status() data = resp.json() assert "token" in data, f"token endpoint did not return a token: {data.keys()}" diff --git a/tests/test_media.py b/tests/test_media.py index eee333f..6270d98 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -17,7 +17,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer import httpx import pytest -from conftest import TOKEN_URL, docker_exec +from conftest import TOKEN_URL, docker_exec, IS_LOCAL_DEPLOYMENT MEDIA_API = "/media/api" TEST_ROOM = f"media-test-{uuid.uuid4().hex[:8]}" @@ -200,7 +200,12 @@ def test_media_stream_proxy_and_range(client): def test_media_files_static_range_support(client): - """Downloaded tracks are nginx static files with native Range support.""" + """Downloaded tracks are nginx static files with native Range support. + + Writes a probe file into the repo's ./media bind mount, so it only runs + when the test host IS the deployment host (local dev deployment).""" + if not IS_LOCAL_DEPLOYMENT: + pytest.skip("writes to the repo ./media bind mount — local deployments only") import pathlib marker = uuid.uuid4().hex[:8] @@ -222,7 +227,11 @@ def test_media_files_static_range_support(client): def test_media_library_persists_in_volume(client): - """The library JSON is written to the ./media bind mount (permanent).""" + """The library JSON is written to the ./media bind mount (permanent). + + Reads the repo's ./media directory — local deployments only.""" + if not IS_LOCAL_DEPLOYMENT: + pytest.skip("reads the repo ./media bind mount — local deployments only") import pathlib upstream, url = _serve_fake_audio(b"\x00" * 16) @@ -271,7 +280,7 @@ def test_ui_media_styles_serve(client): def test_token_response_includes_room(): - resp = httpx.post(TOKEN_URL, json={}, timeout=10) + resp = httpx.post(TOKEN_URL, json={}, timeout=10, verify=False) assert resp.status_code == 200 assert resp.json()["room"].startswith("voice-") @@ -279,7 +288,7 @@ def test_token_response_includes_room(): # ── Hope's media MCP toolset ──────────────────────────────────────────────── MEDIA_MCP_ENV = { - "MEDIA_SERVER_URL": "http://127.0.0.1:8092", + "MEDIA_SERVER_URL": "http://127.0.0.1:8093", "MEDIA_ROOM": "mcp-test-room", } @@ -334,7 +343,7 @@ def test_mcp_media_resume_without_load(mcp_client_factory, client): ) text, is_error = mcp_client_factory( "media_mcp.py", "resume_music", {}, env={ - "MEDIA_SERVER_URL": "http://127.0.0.1:8092", + "MEDIA_SERVER_URL": "http://127.0.0.1:8093", "MEDIA_ROOM": "mcp-empty-room", } ) diff --git a/tests/test_web_ui.py b/tests/test_web_ui.py index 15348c6..50fba3d 100644 --- a/tests/test_web_ui.py +++ b/tests/test_web_ui.py @@ -54,8 +54,11 @@ def test_livekit_client_bundle_serves(client): def test_token_endpoint_direct(): - """Token server on 127.0.0.1:8091 (host network mode).""" - resp = httpx.post("http://127.0.0.1:8091/token", json={}, timeout=10) + """Token endpoint via TOKEN_URL (nginx path on remote deployments, + loopback :8091 when VOICE_TOKEN_URL points at the deployment host).""" + import conftest + + resp = httpx.post(conftest.TOKEN_URL, json={}, timeout=10, verify=False) assert resp.status_code == 200 data = resp.json() assert "token" in data and data["token"].count(".") == 2 # JWT shape diff --git a/web/media_server.py b/web/media_server.py index dee26bf..19190ad 100644 --- a/web/media_server.py +++ b/web/media_server.py @@ -49,7 +49,10 @@ JOBS_DIR = MEDIA_DIR / "jobs" LIBRARY_FILE = MEDIA_DIR / "library.json" COOKIES_FILE = os.environ.get("YTDLP_COOKIES", "") MEDIA_HOST = os.environ.get("MEDIA_SERVER_HOST", "127.0.0.1") -MEDIA_PORT = int(os.environ.get("MEDIA_SERVER_PORT", "8092")) +# 8093, NOT 8092: 8092 is occupied by an Apache/YOURLS stack on the Unraid +# deployment host (xNAS) — a bind conflict there looks like a healthy +# deployment because the healthcheck curl (without -f) accepts any response. +MEDIA_PORT = int(os.environ.get("MEDIA_SERVER_PORT", "8093")) DOWNLOAD_TIMEOUT = int(os.environ.get("MEDIA_DOWNLOAD_TIMEOUT", "600")) SEARCH_TIMEOUT = int(os.environ.get("MEDIA_SEARCH_TIMEOUT", "45"))