feat: media player API, web player panel, and Hope music MCP
- web/media_server.py: FastAPI service owning the permanent library (library.json + ./media volume), yt-dlp YouTube Music search, queued download jobs with mp3 transcode (ffmpeg), per-room player desired state, and an HTTPS proxy for external live-stream URLs. - agent/media_mcp.py: Hope tools — search_music, play_song, offer_song_choices, pause/resume/stop/skip/seek, now_playing, list_library, add_stream. - agent/agent.py: media toolset wiring + conversation rule (music playing disables the agent mic; re-engaging pauses music). - web UI: slide-out Music panel (search, suggestions, library, live stream add, play/pause, ±10s, seek, volume) with a reconciling poller and media_state data-channel reports. - token server returns the minted room name; nginx serves /media/files as static (native Range) and proxies /media/api to the media server. - supervisord runs media-server; compose mounts ./media and adds ffmpeg to the image.
This commit is contained in:
1 parent
a4ae96f01b
commit
04c4a53594
16 files changed
+1876
-12
No files matched your search
+64
-2
@@ -196,6 +196,23 @@ SYSTEM_PROMPT = textwrap.dedent("""\
|
||||
you receive it, share the findings naturally in your conversational style.
|
||||
You can have multiple tasks running at once.
|
||||
|
||||
# Music (media player in the web UI)
|
||||
You can play music on the user's web player. For any music request, call
|
||||
search_music first, then:
|
||||
- One clearly-right result → play_song with its video_id. Announce it in
|
||||
one short sentence, then stop talking — playback pauses the
|
||||
conversation until the user comes back to you.
|
||||
- Ambiguous results → offer_song_choices with the top 2-3 options, name
|
||||
them briefly, and let the user pick by voice or in the player panel.
|
||||
Never start playback on a guess.
|
||||
The library is permanent: list_library shows what is already downloaded,
|
||||
and those tracks play instantly without searching. If the user names a
|
||||
library track exactly, just play it. Control playback with pause_music,
|
||||
resume_music, skip_song and seek_music; check what's on with now_playing.
|
||||
add_stream registers a live radio or direct audio URL as a library track.
|
||||
While music is playing you cannot hear the user — they will come back to
|
||||
you when they want to talk again, and the music pauses automatically.
|
||||
|
||||
# Skills
|
||||
You have a skill library of learned procedures. Use skill_recall when you need
|
||||
to perform a task you've done before — it will give you the steps. When you
|
||||
@@ -624,6 +641,27 @@ def build_mcp_toolsets(user_profile: dict, task_dir: str) -> list[mcp.MCPToolset
|
||||
)
|
||||
logger.info("Dispatch MCP toolset enabled (%s)", task_dir)
|
||||
|
||||
# Media tools talk to the local media server; the room name scopes player
|
||||
# state so concurrent sessions don't fight over one virtual player.
|
||||
media_server_url = os.environ.get("MEDIA_SERVER_URL", "http://127.0.0.1:8092")
|
||||
media_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media_mcp.py")
|
||||
toolsets.append(
|
||||
mcp.MCPToolset(
|
||||
id="media",
|
||||
mcp_server=mcp.MCPServerStdio(
|
||||
command=python_bin,
|
||||
args=[media_mcp_script],
|
||||
env={
|
||||
**os.environ,
|
||||
"MEDIA_SERVER_URL": media_server_url,
|
||||
"MEDIA_ROOM": os.path.basename(task_dir.rstrip("/")) or "default",
|
||||
},
|
||||
client_session_timeout_seconds=60,
|
||||
),
|
||||
)
|
||||
)
|
||||
logger.info("Media MCP toolset enabled (%s)", media_server_url)
|
||||
|
||||
extra = os.environ.get("EXTRA_MCP_SERVERS", "")
|
||||
if extra:
|
||||
try:
|
||||
@@ -740,9 +778,22 @@ async def handle_job(ctx: JobContext) -> None:
|
||||
|
||||
# Listen for conversation controls (voice selection and voice/text mode).
|
||||
# Defined here so it captures this job's session via closure.
|
||||
# Music and conversation are mutually exclusive by design: while a track
|
||||
# plays the agent's ears are off; re-engaging the conversation pauses the
|
||||
# music, and ending the conversation lets the user resume it.
|
||||
session_mode = {"value": "voice"} # UI voice/text switch
|
||||
media_playback = {"value": "stopped"} # UI-reported player state
|
||||
|
||||
def _apply_listening() -> None:
|
||||
listening = (
|
||||
session_mode["value"] == "voice"
|
||||
and media_playback["value"] != "playing"
|
||||
)
|
||||
session.input.set_audio_enabled(listening)
|
||||
|
||||
def _on_room_data(packet) -> None:
|
||||
topic = getattr(packet, "topic", None)
|
||||
if topic and topic not in ("voice-control", "conversation-control"):
|
||||
if topic and topic not in ("voice-control", "conversation-control", "media-control"):
|
||||
return
|
||||
try:
|
||||
msg = json.loads(packet.data.decode("utf-8"))
|
||||
@@ -755,11 +806,22 @@ async def handle_job(ctx: JobContext) -> None:
|
||||
return
|
||||
voice_enabled = mode == "voice"
|
||||
logger.info("Switching conversation mode to %s", mode)
|
||||
session_mode["value"] = mode
|
||||
if not voice_enabled:
|
||||
# interrupt() returns an already-scheduled Future, not a coroutine.
|
||||
session.interrupt()
|
||||
session.input.set_audio_enabled(voice_enabled)
|
||||
session.output.set_audio_enabled(voice_enabled)
|
||||
_apply_listening()
|
||||
return
|
||||
|
||||
if msg.get("type") == "media_state":
|
||||
state = msg.get("state")
|
||||
if state not in ("playing", "paused", "stopped"):
|
||||
return
|
||||
logger.info("Media playback -> %s (agent listening: %s)",
|
||||
state, session_mode["value"] == "voice" and state != "playing")
|
||||
media_playback["value"] = state
|
||||
_apply_listening()
|
||||
return
|
||||
|
||||
if msg.get("type") == "set_voice":
|
||||
|
||||
Reference in new issue
Block a user