feat: progressive music streaming, library-first search, player fixes

- Stream downloads as they grow: raw best-audio (no mp3 transcode) served
  chunked from the stream endpoint mid-download, then with native Range
  once complete. Songs play within seconds of hitting Play.
- Fix searched songs never starting: reconcileMedia now wires the stream
  URL when a track that was preparing gains one; audio_url is stable for
  the whole lifecycle so the player never reloads mid-playback.
- Progress bars now parse yt-dlp stderr (stdout never carried [download]).
- Prefer downloaded songs: /media/api/search merges ready library matches
  ahead of YouTube results; search_music/play_song tooling and the agent
  prompt tell Hope to always pick in_library=true.
- Mute button turns solid red while the mic is muted.

web/media_server.py

@@ -4,32 +4,36 @@
 A small FastAPI service that owns the permanent music library and player
 coordination:
 
-- Library: downloaded YouTube Music tracks (mp3, transcoded via yt-dlp +
-  ffmpeg) and user-added live audio stream URLs, persisted in library.json.
+- Library: downloaded YouTube Music tracks (raw best-audio files, saved by
+  yt-dlp with no transcode) and user-added live audio stream URLs, persisted
+  in library.json.
 - Search: yt-dlp ``ytsearch`` (YouTube Music pages share video ids), used by
-  Hope's media MCP toolset and by the web UI search box.
+  Hope's media MCP toolset and by the web UI search box. Ready library tracks
+  that match the query are returned first so Hope prefers a downloaded copy.
 - Download jobs: one-at-a-time worker thread; progress is parsed from
-  yt-dlp stdout and surfaced through GET /media/api/state.
+  yt-dlp stderr and surfaced through GET /media/api/state. Playback starts
+  while the file is still growing (progressive streaming), and the finished
+  file stays in the library for instant replay.
 - Player state: per-room desired state (loaded track, playing/paused/stopped,
   seek hints). The browser is the actual player and reconciles against this;
   it reports real playback state back so Hope's voice tools can answer
   "what's playing".
 
 Security posture matches the token server: bind 127.0.0.1 only, reachability
-exclusively through the nginx HTTPS proxy (/media/api/). File bytes for
-downloaded tracks are served as nginx static files (/media/files/, native
-HTTP Range support for seeking); this service only proxies external stream
-URLs (mixed-content: an http:// radio URL would be blocked by the browser,
+exclusively through the nginx HTTPS proxy (/media/api/). Growing downloads
+are streamed by this service (chunked, no Content-Length); finished files are
+served here too with native Range support for seeking. Live stream URLs are
+proxied (mixed-content: an http:// radio URL would be blocked by the browser,
 so it is fetched server-side and re-served over HTTPS).
 """
 from __future__ import annotations
 
+import asyncio
 import json
 import os
 import re
 import subprocess
 import sys
-import tempfile
 import threading
 import time
 import uuid
@@ -40,7 +44,7 @@ from typing import Any
 import httpx
 import uvicorn
 from fastapi import FastAPI, HTTPException, Request
-from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse
+from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 
 MEDIA_DIR = Path(os.environ.get("MEDIA_DIR", "/data/media"))
 FILES_DIR = MEDIA_DIR / "files"
@@ -107,8 +111,11 @@ def _bump_version() -> int:
 def _track_dict(track: dict) -> dict:
     out = dict(track)
     if track["kind"] == "youtube":
-        if track.get("status") == "ready":
-            out["audio_url"] = f"/media/files/{track['id']}.mp3"
+        # The stream endpoint serves the growing file during download and the
+        # finished file afterwards; the URL is stable across the transition
+        # so the browser never has to reload mid-playback.
+        if track.get("status") in ("downloading", "ready"):
+            out["audio_url"] = f"/media/api/stream/{track['id']}"
         out["art_url"] = (
             f"/media/thumbs/{track['id']}.jpg"
             if (THUMBS_DIR / f"{track['id']}.jpg").exists()
@@ -179,6 +186,101 @@ def _ytdlp_base_args() -> list[str]:
 
 _PROGRESS_RE = re.compile(r"\[download\]\s+([\d.]+)%")
 
+# Audio containers yt-dlp can produce for a raw best-audio download.
+_AUDIO_EXTS = {
+    ".webm", ".opus", ".ogg", ".m4a", ".mp4", ".m4b", ".aac",
+    ".mp3", ".mka", ".flac", ".wav",
+}
+_AUDIO_MIME = {
+    ".webm": "audio/webm",
+    ".opus": "audio/opus",
+    ".ogg": "audio/ogg",
+    ".m4a": "audio/mp4",
+    ".mp4": "audio/mp4",
+    ".m4b": "audio/mp4",
+    ".aac": "audio/aac",
+    ".mp3": "audio/mpeg",
+    ".mka": "audio/x-matroska",
+    ".flac": "audio/flac",
+    ".wav": "audio/wav",
+}
+
+
+def _media_type_for(name: str) -> str:
+    if name.endswith(".part"):
+        name = name[:-5]
+    return _AUDIO_MIME.get(Path(name).suffix.lower(), "audio/mpeg")
+
+
+def _youtube_media_file(track_id: str) -> Path | None:
+    """The on-disk audio file for a youtube track, if any.
+
+    During an active download yt-dlp writes ``{id}.{ext}.part`` and renames it
+    to ``{id}.{ext}`` on completion — return whichever currently exists (the
+    partial file wins, thumbnails and other sidecars are ignored).
+    """
+    complete: Path | None = None
+    for p in FILES_DIR.glob(f"{track_id}.*"):
+        suffix = p.suffix.lower()
+        if suffix == ".part":
+            if p.stat().st_size > 0:
+                return p
+        elif suffix in _AUDIO_EXTS:
+            if complete is None or p.name < complete.name:
+                complete = p
+    return complete
+
+
+async def _wait_for_media_file(track_id: str, timeout: float = 15.0) -> Path | None:
+    """Wait for yt-dlp to create the output file after a play request."""
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        path = _youtube_media_file(track_id)
+        if path is not None:
+            return path
+        await asyncio.sleep(0.5)
+    return None
+
+
+async def _growing_file(track_id: str, job: dict, chunk_size: int = 64 * 1024):
+    """Yield bytes from a file that yt-dlp is still writing.
+
+    The response stays open with no Content-Length until the download
+    finishes, so the browser plays the track as it grows. The open descriptor
+    keeps reading the old inode across yt-dlp's .part -> final rename, and the
+    path is re-resolved if the file disappears between resolution and open.
+    """
+    fd = -1
+    pos = 0
+    stall = 0
+    try:
+        while True:
+            if fd == -1:
+                path = _youtube_media_file(track_id)
+                if path is not None:
+                    try:
+                        fd = os.open(path, os.O_RDONLY)
+                    except FileNotFoundError:
+                        fd = -1  # renamed between glob and open — retry
+            if fd != -1:
+                data = await asyncio.to_thread(os.pread, fd, chunk_size, pos)
+                if data:
+                    pos += len(data)
+                    stall = 0
+                    yield data
+                    continue
+            with _lock:
+                active = job.get("stage") == "downloading"
+            if not active:
+                break
+            stall += 1
+            if stall > 60:  # ~30s with no new bytes — stop waiting
+                break
+            await asyncio.sleep(0.5)
+    finally:
+        if fd != -1:
+            os.close(fd)
+
 
 def _run_search(query: str, limit: int) -> list[dict]:
     """Flat ytsearch lookup — cheap, no download, works anonymously."""
@@ -247,15 +349,18 @@ def _fail_job(job: dict, track: dict, message: str) -> None:
 
 
 def _download_worker(job: dict, track: dict, video_id: str) -> None:
-    """Runs in the single-worker executor: yt-dlp download + mp3 transcode."""
-    target = FILES_DIR / f"{track['id']}.mp3"
+    """Runs in the single-worker executor: raw best-audio download.
+
+    No mp3 transcode — the audio container is streamed straight into the
+    library so playback can begin while the file is still growing (the stream
+    endpoint serves the partial file), and the finished file stays playable
+    offline. Thumbnail handling is unchanged.
+    """
     cmd = _ytdlp_base_args() + [
         "-f", "bestaudio/best",
-        "-x", "--audio-format", "mp3", "--audio-quality", "192K",
-        "--embed-thumbnail", "--add-metadata",
-        "--write-thumbnail", "--convert-thumbnails", "jpg",
         "--newline", "--progress",
         "--print-json",
+        "--write-thumbnail", "--convert-thumbnails", "jpg",
         "--retries", "3", "--fragment-retries", "3",
         "-P", str(FILES_DIR),
         "-o", f"{track['id']}.%(ext)s",
@@ -263,46 +368,52 @@ def _download_worker(job: dict, track: dict, video_id: str) -> None:
     ]
     job["stage"] = "downloading"
     info: dict = {}
-    # stderr goes to a temp file: reading it from a PIPE while the stdout
-    # loop blocks could deadlock if yt-dlp ever fills the stderr buffer.
-    with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as errfile:
-        try:
-            proc = subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=errfile,
-                text=True, errors="replace",
-            )
-            deadline = time.monotonic() + DOWNLOAD_TIMEOUT
-            assert proc.stdout is not None
-            for line in proc.stdout:
-                if time.monotonic() > deadline:
-                    proc.kill()
-                    proc.wait(timeout=30)
-                    raise RuntimeError(f"Download exceeded {DOWNLOAD_TIMEOUT}s")
-                line = line.strip()
-                m = _PROGRESS_RE.search(line)
-                if m:
-                    job["progress"] = min(85.0, float(m.group(1)) * 0.85)
-                    _persist_job(job)
-                elif line.startswith(("[ExtractAudio]", "[Metadata]", "[EmbedThumbnail]")):
-                    job["stage"] = "transcoding"
-                    job["progress"] = max(job["progress"], 86.0)
-                    _persist_job(job)
-                elif line.startswith("{"):
-                    try:
-                        info = json.loads(line)
-                    except json.JSONDecodeError:
-                        pass
-            proc.wait(timeout=30)
-            if proc.returncode != 0:
-                errfile.seek(0)
-                stderr_tail = (errfile.read() or "").strip().splitlines()
-                detail = stderr_tail[-1] if stderr_tail else f"exit code {proc.returncode}"
-                raise RuntimeError(detail)
-            if not target.exists():
-                raise RuntimeError("yt-dlp finished but no mp3 was produced")
-        except Exception as exc:  # noqa: BLE001 — any failure marks the track failed
-            _fail_job(job, track, str(exc))
-            return
+    # Progress lines go to stderr, the metadata JSON to stdout. Drain stderr
+    # in a thread so the stdout read loop can never deadlock the pipe; the
+    # tail is kept for the failure detail.
+    stderr_tail: list[str] = []
+    proc = None
+
+    def _drain_stderr() -> None:
+        assert proc is not None and proc.stderr is not None
+        for line in proc.stderr:
+            stderr_tail.append(line)
+            if len(stderr_tail) > 200:
+                stderr_tail.pop(0)
+            m = _PROGRESS_RE.search(line)
+            if m:
+                job["progress"] = min(99.0, float(m.group(1)))
+                _persist_job(job)
+
+    try:
+        proc = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            text=True, errors="replace",
+        )
+        threading.Thread(target=_drain_stderr, daemon=True).start()
+        deadline = time.monotonic() + DOWNLOAD_TIMEOUT
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            if time.monotonic() > deadline:
+                proc.kill()
+                proc.wait(timeout=30)
+                raise RuntimeError(f"Download exceeded {DOWNLOAD_TIMEOUT}s")
+            line = line.strip()
+            if line.startswith("{"):
+                try:
+                    info = json.loads(line)
+                except json.JSONDecodeError:
+                    pass
+        proc.wait(timeout=30)
+        if proc.returncode != 0:
+            detail = stderr_tail[-1] if stderr_tail else f"exit code {proc.returncode}"
+            raise RuntimeError(detail.strip())
+        media = _youtube_media_file(track["id"])
+        if media is None:
+            raise RuntimeError("yt-dlp finished but no audio file was produced")
+    except Exception as exc:  # noqa: BLE001 — any failure marks the track failed
+        _fail_job(job, track, str(exc))
+        return
 
     thumb_src = FILES_DIR / f"{track['id']}.jpg"
     try:
@@ -317,6 +428,7 @@ def _download_worker(job: dict, track: dict, video_id: str) -> None:
     with _lock:
         track["status"] = "ready"
         track["error"] = None
+        track["ext"] = media.suffix.lstrip(".").lower()
         if info:
             if info.get("title"):
                 track["title"] = info["title"]
@@ -364,11 +476,11 @@ def get_library() -> dict:
 def _delete_track_files(track: dict) -> None:
     if track["kind"] != "youtube":
         return
-    for path in (
-        FILES_DIR / f"{track['id']}.mp3",
-        FILES_DIR / f"{track['id']}.jpg",
-        THUMBS_DIR / f"{track['id']}.jpg",
-    ):
+    paths = [
+        FILES_DIR / f"{track['id']}.{ext}" for ext in
+        (s.lstrip(".") for s in _AUDIO_EXTS)
+    ] + [FILES_DIR / f"{track['id']}.jpg", THUMBS_DIR / f"{track['id']}.jpg"]
+    for path in paths:
         try:
             path.unlink(missing_ok=True)
         except OSError:
@@ -394,6 +506,39 @@ def delete_track(track_id: str) -> dict:
 
 # ── API: search + play + control ────────────────────────────────────────────
 
+def _library_search(query: str, limit: int) -> list[dict]:
+    """Ready library tracks whose title matches the query.
+
+    These are merged in front of the YouTube results so Hope prefers an
+    already-downloaded copy over fetching a new one.
+    """
+    q = query.strip().lower()
+    tokens = {t for t in re.split(r"\W+", q) if len(t) > 2}
+    hits: list[dict] = []
+    if not q:
+        return hits
+    for track in _library["tracks"].values():
+        if track.get("kind") != "youtube" or track.get("status") != "ready":
+            continue
+        title = (track.get("title") or "").strip().lower()
+        if not title:
+            continue
+        title_tokens = {t for t in re.split(r"\W+", title) if t}
+        if not (q in title or title in q or (tokens and tokens <= title_tokens)):
+            continue
+        hits.append({
+            "video_id": track["id"],
+            "track_id": track["id"],
+            "title": track.get("title") or track["id"],
+            "channel": track.get("channel") or "",
+            "duration": track.get("duration"),
+            "view_count": None,
+            "in_library": True,
+            "library_status": "ready",
+        })
+    return hits[:limit]
+
+
 @app.post("/media/api/search")
 async def search(request: Request) -> dict:
     body = await request.json()
@@ -404,7 +549,11 @@ async def search(request: Request) -> dict:
         limit = max(1, min(int(body.get("limit", 6)), 15))
     except (TypeError, ValueError):
         raise HTTPException(400, "limit must be an integer")
-    return {"query": query, "results": _run_search(query, limit)}
+    local = _library_search(query, limit)
+    remote = _run_search(query, limit)
+    seen = {r["video_id"] for r in local}
+    merged = local + [r for r in remote if r["video_id"] not in seen]
+    return {"query": query, "results": merged[:limit]}
 
 
 def _resolve_play_target(body: dict) -> dict:
@@ -588,13 +737,32 @@ _HOP_HEADERS = ("content-type", "content-range", "accept-ranges", "content-lengt
 async def stream_proxy(track_id: str, request: Request):
     with _lock:
         track = _get_track(track_id)
+        job = next(
+            (j for j in _jobs.values()
+             if j["track_id"] == track_id and j["stage"] == "downloading"),
+            None,
+        )
     if not track:
         raise HTTPException(404, "unknown track")
+
     if track["kind"] == "youtube":
-        # Downloaded tracks are nginx static files (native Range support).
-        if track.get("status") == "ready":
-            return RedirectResponse(f"/media/files/{track_id}.mp3")
-        raise HTTPException(409, f"track is {track.get('status')}, not ready")
+        # Progressive download: an active download is streamed in real time
+        # (chunked, no Content-Length) so playback starts while the file
+        # grows; a finished file is served directly with native Range support.
+        path = _youtube_media_file(track_id)
+        if path is None and job is not None:
+            path = await _wait_for_media_file(track_id)
+        if path is None:
+            raise HTTPException(
+                404, f"audio file not available (track is {track.get('status')})"
+            )
+        if job is not None:
+            return StreamingResponse(
+                _growing_file(track_id, job),
+                media_type=_media_type_for(path.name),
+                headers={"Cache-Control": "no-store"},
+            )
+        return FileResponse(path, media_type=_media_type_for(path.name))
 
     range_header = request.headers.get("range")
     headers = {"Range": range_header} if range_header else {}