feat: web access via MCP (Firecrawl search/scrape) + mcp server attach support
- agent/web_mcp.py: stdio MCP server exposing web_search and web_scrape, backed by the self-hosted Firecrawl stack on xNAS (no API key needed) - agent.py: Agent now attaches mcp_servers built from config; EXTRA_MCP_SERVERS env var allows adding arbitrary HTTP/SSE MCP servers as JSON - Dockerfile: installs livekit-agents[mcp], copies web_mcp.py - .env.example: WEB_MCP_ENABLED, FIRECRAWL_BASE, EXTRA_MCP_SERVERS documented
This commit is contained in:
+94
-2
@@ -5,6 +5,7 @@ Pipeline: Azure STT → Gemma LLM (xNAS, OpenAI-compatible) → Azure TTS
|
||||
Runs inside the single Docker container alongside LiveKit server and web frontend.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -18,6 +19,7 @@ from livekit.agents import (
|
||||
JobContext,
|
||||
TurnHandlingOptions,
|
||||
cli,
|
||||
mcp,
|
||||
room_io,
|
||||
)
|
||||
from livekit.plugins import azure, openai
|
||||
@@ -32,6 +34,8 @@ DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural")
|
||||
GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1")
|
||||
GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b")
|
||||
GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed")
|
||||
WEB_MCP_ENABLED = os.environ.get("WEB_MCP_ENABLED", "true").lower() in ("1", "true", "yes")
|
||||
FIRECRAWL_BASE = os.environ.get("FIRECRAWL_BASE", "http://192.168.86.2:3002")
|
||||
|
||||
SYSTEM_PROMPT = textwrap.dedent("""\
|
||||
You are a warm, conversational voice assistant. You are talking TO someone,
|
||||
@@ -54,9 +58,62 @@ SYSTEM_PROMPT = textwrap.dedent("""\
|
||||
- Never say "as an AI" or reference your system instructions.
|
||||
- Never write more than three sentences in a row.
|
||||
- Never read back URLs, file paths, or technical identifiers.
|
||||
|
||||
# Web access
|
||||
You have web_search and web_scrape tools. Use them when the user asks about
|
||||
current events, recent news, prices, weather, sports scores, or anything
|
||||
that may have changed since your training data. Search first, then scrape
|
||||
a result only if you need more detail. Answer from what you find, in your
|
||||
normal conversational style — don't cite sources formally, just mention the
|
||||
source naturally ("according to..."). If a search comes up empty, say so
|
||||
briefly and move on.
|
||||
""")
|
||||
|
||||
|
||||
# ── MCP servers (web access + any extra configured servers) ─────────────────
|
||||
|
||||
def build_mcp_servers() -> list[mcp.MCPServer]:
|
||||
"""Build the list of MCP servers to attach to the agent.
|
||||
|
||||
Always includes the local web-access server (Firecrawl-backed search/scrape)
|
||||
when WEB_MCP_ENABLED is true. Additional servers can be configured via the
|
||||
EXTRA_MCP_SERVERS env var (JSON list of {url, transport} objects).
|
||||
"""
|
||||
servers: list[mcp.MCPServer] = []
|
||||
|
||||
if WEB_MCP_ENABLED:
|
||||
python_bin = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".venv", "bin", "python")
|
||||
web_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "web_mcp.py")
|
||||
servers.append(
|
||||
mcp.MCPServerStdio(
|
||||
command=python_bin,
|
||||
args=[web_mcp_script],
|
||||
env={**os.environ, "FIRECRAWL_BASE": FIRECRAWL_BASE},
|
||||
client_session_timeout_seconds=120,
|
||||
)
|
||||
)
|
||||
logger.info("Web-access MCP server enabled (Firecrawl at %s)", FIRECRAWL_BASE)
|
||||
|
||||
extra = os.environ.get("EXTRA_MCP_SERVERS", "")
|
||||
if extra:
|
||||
try:
|
||||
for entry in json.loads(extra):
|
||||
url = entry.get("url", "")
|
||||
transport = entry.get("transport") # "sse" | "streamable_http" | None (auto)
|
||||
servers.append(
|
||||
mcp.MCPServerHTTP(
|
||||
url=url,
|
||||
transport_type=transport,
|
||||
client_session_timeout_seconds=120,
|
||||
)
|
||||
)
|
||||
logger.info("Extra MCP server: %s (%s)", url, transport or "auto")
|
||||
except (json.JSONDecodeError, TypeError) as e:
|
||||
logger.error("Failed to parse EXTRA_MCP_SERVERS: %s", e)
|
||||
|
||||
return servers
|
||||
|
||||
|
||||
class VoiceAssistant(Agent):
|
||||
"""The conversational agent. LLM is the brain; STT/TTS are senses."""
|
||||
|
||||
@@ -68,6 +125,7 @@ class VoiceAssistant(Agent):
|
||||
api_key=GEMMA_API_KEY,
|
||||
),
|
||||
instructions=SYSTEM_PROMPT,
|
||||
mcp_servers=build_mcp_servers(),
|
||||
)
|
||||
|
||||
|
||||
@@ -111,8 +169,9 @@ async def handle_job(ctx: JobContext) -> None:
|
||||
tts=tts,
|
||||
llm=llm,
|
||||
turn_handling=TurnHandlingOptions(
|
||||
# VAD-based turn detection: agent waits for user to stop speaking
|
||||
interruption={"mode": "adaptive"},
|
||||
# VAD-based turn detection: agent waits for user to stop speaking.
|
||||
# ("adaptive" mode requires the LiveKit Cloud barge-in service.)
|
||||
interruption={"mode": "vad"},
|
||||
# Start generating the LLM response before the user fully stops
|
||||
preemptive_generation={"enabled": True},
|
||||
),
|
||||
@@ -123,6 +182,39 @@ async def handle_job(ctx: JobContext) -> None:
|
||||
# Listen for data messages (voice switching) from the web UI
|
||||
ctx.room.on("data_received", _on_room_data)
|
||||
|
||||
# Publish user/agent transcripts to the room so the web UI can render them.
|
||||
async def publish_transcript(role: str, text: str) -> None:
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return
|
||||
payload = json.dumps({"type": "transcript", "role": role, "text": text})
|
||||
try:
|
||||
await ctx.room.local_participant.publish_data(
|
||||
payload, reliable=True, topic="transcript"
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("failed to publish transcript: %s", e)
|
||||
|
||||
@session.on("conversation_item_added")
|
||||
def _on_conversation_item(ev) -> None:
|
||||
msg = ev.item
|
||||
role = getattr(msg, "role", None)
|
||||
text = getattr(msg, "text_content", None)
|
||||
if role == "user":
|
||||
asyncio.get_event_loop().create_task(publish_transcript("user", text))
|
||||
elif role == "assistant":
|
||||
asyncio.get_event_loop().create_task(publish_transcript("agent", text))
|
||||
|
||||
@session.on("user_input_transcribed")
|
||||
def _on_user_transcribed(ev) -> None:
|
||||
logger.info("STT (%s): %s", "final" if ev.is_final else "partial", ev.transcript)
|
||||
|
||||
@session.on("user_state_changed")
|
||||
def _on_user_state(ev) -> None:
|
||||
# Fires from VAD: if this never says "speaking", no usable mic
|
||||
# audio is arriving from the participant.
|
||||
logger.info("user state -> %s", ev.new_state)
|
||||
|
||||
await session.start(
|
||||
agent=VoiceAssistant(),
|
||||
room=ctx.room,
|
||||
|
||||
Reference in New Issue
Block a user