snowspeeder/hope-voice-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: web access via MCP (Firecrawl search/scrape) + mcp server attach support

Browse Source 
- agent/web_mcp.py: stdio MCP server exposing web_search and web_scrape,
  backed by the self-hosted Firecrawl stack on xNAS (no API key needed)
- agent.py: Agent now attaches mcp_servers built from config; EXTRA_MCP_SERVERS
  env var allows adding arbitrary HTTP/SSE MCP servers as JSON
- Dockerfile: installs livekit-agents[mcp], copies web_mcp.py
- .env.example: WEB_MCP_ENABLED, FIRECRAWL_BASE, EXTRA_MCP_SERVERS documented
This commit is contained in:
Shane
2026-08-22 12:29:04 -04:00
parent 6f2b231938
commit 045ddabac7
12 changed files with 452 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/agent.py
+94 -2
View File
@@ -5,6 +5,7 @@ Pipeline: Azure STT → Gemma LLM (xNAS, OpenAI-compatible) → Azure TTS
Runs inside the single Docker container alongside LiveKit server and web frontend.
"""
import asyncio
import json
import logging
import os
@@ -18,6 +19,7 @@ from livekit.agents import (
JobContext,
TurnHandlingOptions,
cli,
mcp,
room_io,
)
from livekit.plugins import azure, openai
@@ -32,6 +34,8 @@ DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural")
GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1")
GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b")
GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed")
WEB_MCP_ENABLED = os.environ.get("WEB_MCP_ENABLED", "true").lower() in ("1", "true", "yes")
FIRECRAWL_BASE = os.environ.get("FIRECRAWL_BASE", "http://192.168.86.2:3002")
SYSTEM_PROMPT = textwrap.dedent("""\
You are a warm, conversational voice assistant. You are talking TO someone,
@@ -54,9 +58,62 @@ SYSTEM_PROMPT = textwrap.dedent("""\
- Never say "as an AI" or reference your system instructions.
- Never write more than three sentences in a row.
- Never read back URLs, file paths, or technical identifiers.
# Web access
You have web_search and web_scrape tools. Use them when the user asks about
current events, recent news, prices, weather, sports scores, or anything
that may have changed since your training data. Search first, then scrape
a result only if you need more detail. Answer from what you find, in your
normal conversational style — don't cite sources formally, just mention the
source naturally ("according to..."). If a search comes up empty, say so
briefly and move on.
""")
# ── MCP servers (web access + any extra configured servers) ─────────────────
def build_mcp_servers() -> list[mcp.MCPServer]:
"""Build the list of MCP servers to attach to the agent.
Always includes the local web-access server (Firecrawl-backed search/scrape)
when WEB_MCP_ENABLED is true. Additional servers can be configured via the
EXTRA_MCP_SERVERS env var (JSON list of {url, transport} objects).
"""
servers: list[mcp.MCPServer] = []
if WEB_MCP_ENABLED:
python_bin = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".venv", "bin", "python")
web_mcp_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "web_mcp.py")
servers.append(
mcp.MCPServerStdio(
command=python_bin,
args=[web_mcp_script],
env={**os.environ, "FIRECRAWL_BASE": FIRECRAWL_BASE},
client_session_timeout_seconds=120,
)
)
logger.info("Web-access MCP server enabled (Firecrawl at %s)", FIRECRAWL_BASE)
extra = os.environ.get("EXTRA_MCP_SERVERS", "")
if extra:
try:
for entry in json.loads(extra):
url = entry.get("url", "")
transport = entry.get("transport") # "sse" | "streamable_http" | None (auto)
servers.append(
mcp.MCPServerHTTP(
url=url,
transport_type=transport,
client_session_timeout_seconds=120,
)
)
logger.info("Extra MCP server: %s (%s)", url, transport or "auto")
except (json.JSONDecodeError, TypeError) as e:
logger.error("Failed to parse EXTRA_MCP_SERVERS: %s", e)
return servers
class VoiceAssistant(Agent):
"""The conversational agent. LLM is the brain; STT/TTS are senses."""
@@ -68,6 +125,7 @@ class VoiceAssistant(Agent):
api_key=GEMMA_API_KEY,
),
instructions=SYSTEM_PROMPT,
mcp_servers=build_mcp_servers(),
)
@@ -111,8 +169,9 @@ async def handle_job(ctx: JobContext) -> None:
tts=tts,
llm=llm,
turn_handling=TurnHandlingOptions(
# VAD-based turn detection: agent waits for user to stop speaking
interruption={"mode": "adaptive"},
# VAD-based turn detection: agent waits for user to stop speaking.
# ("adaptive" mode requires the LiveKit Cloud barge-in service.)
interruption={"mode": "vad"},
# Start generating the LLM response before the user fully stops
preemptive_generation={"enabled": True},
),
@@ -123,6 +182,39 @@ async def handle_job(ctx: JobContext) -> None:
# Listen for data messages (voice switching) from the web UI
ctx.room.on("data_received", _on_room_data)
# Publish user/agent transcripts to the room so the web UI can render them.
async def publish_transcript(role: str, text: str) -> None:
text = (text or "").strip()
if not text:
return
payload = json.dumps({"type": "transcript", "role": role, "text": text})
try:
await ctx.room.local_participant.publish_data(
payload, reliable=True, topic="transcript"
)
except Exception as e: # noqa: BLE001
logger.warning("failed to publish transcript: %s", e)
@session.on("conversation_item_added")
def _on_conversation_item(ev) -> None:
msg = ev.item
role = getattr(msg, "role", None)
text = getattr(msg, "text_content", None)
if role == "user":
asyncio.get_event_loop().create_task(publish_transcript("user", text))
elif role == "assistant":
asyncio.get_event_loop().create_task(publish_transcript("agent", text))
@session.on("user_input_transcribed")
def _on_user_transcribed(ev) -> None:
logger.info("STT (%s): %s", "final" if ev.is_final else "partial", ev.transcript)
@session.on("user_state_changed")
def _on_user_state(ev) -> None:
# Fires from VAD: if this never says "speaking", no usable mic
# audio is arriving from the participant.
logger.info("user state -> %s", ev.new_state)
await session.start(
agent=VoiceAssistant(),
room=ctx.room,
agent/pyproject.toml
+1
View File
@@ -10,6 +10,7 @@ requires-python = ">=3.10,<3.15"
dependencies = [
"livekit-agents~=1.7",
"livekit-agents[mcp]~=1.7",
"livekit-plugins-azure~=1.7",
"livekit-plugins-openai~=1.7",
"python-dotenv",
agent/web_mcp.py
+108
View File
@@ -0,0 +1,108 @@
"""Web access MCP server — exposes Firecrawl search + scrape as MCP tools.
Runs over stdio inside the voice container. The agent attaches it via
MCPServerStdio, so the LLM can call web_search / web_scrape during a
conversation to look things up in real time.
Backed by the self-hosted Firecrawl stack on xNAS (no API key needed):
- POST {FIRECRAWL_BASE}/v1/search -> ranked results with title/description
- POST {FIRECRAWL_BASE}/v1/scrape -> page content as markdown
Config (env vars, all optional):
FIRECRAWL_BASE default http://192.168.86.2:3002
FIRECRAWL_API_KEY only needed if the Firecrawl instance requires auth
WEB_SEARCH_LIMIT default 5 results per search
"""
from __future__ import annotations
import json
import os
import httpx
from mcp.server.fastmcp import FastMCP
FIRECRAWL_BASE = os.environ.get("FIRECRAWL_BASE", "http://192.168.86.2:3002").rstrip("/")
FIRECRAWL_API_KEY = os.environ.get("FIRECRAWL_API_KEY", "")
WEB_SEARCH_LIMIT = int(os.environ.get("WEB_SEARCH_LIMIT", "5"))
mcp = FastMCP("web-access")
def _headers() -> dict[str, str]:
h = {"Content-Type": "application/json"}
if FIRECRAWL_API_KEY:
h["Authorization"] = f"Bearer {FIRECRAWL_API_KEY}"
return h
@mcp.tool()
async def web_search(query: str, limit: int | None = None) -> str:
"""Search the web and return ranked results with titles, URLs, and descriptions.
Use this to find current information, news, facts, or sources about a topic.
Returns a compact text summary — not raw JSON.
"""
limit = min(limit or WEB_SEARCH_LIMIT, 10)
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
f"{FIRECRAWL_BASE}/v1/search",
headers=_headers(),
json={"query": query, "limit": limit},
)
resp.raise_for_status()
data = resp.json()
if not data.get("success"):
return f"Search failed: {data.get('error', 'unknown error')}"
results = data.get("data", [])
if not results:
return f"No results found for: {query}"
lines = [f"Web search results for: {query}", ""]
for i, r in enumerate(results, 1):
title = (r.get("title") or "(no title)").strip()
url = (r.get("url") or "").strip()
desc = (r.get("description") or "").strip().replace("\n", " ")
lines.append(f"{i}. {title}")
if desc:
lines.append(f" {desc[:300]}")
if url:
lines.append(f" URL: {url}")
return "\n".join(lines)
@mcp.tool()
async def web_scrape(url: str, max_chars: int = 8000) -> str:
"""Fetch a web page and return its content as readable markdown.
Use this after web_search to read the full content of a promising result.
The content is truncated to max_chars (default 8000) to stay within context.
"""
async with httpx.AsyncClient(timeout=90) as client:
resp = await client.post(
f"{FIRECRAWL_BASE}/v1/scrape",
headers=_headers(),
json={"url": url, "formats": ["markdown"], "timeout": 60000},
)
resp.raise_for_status()
data = resp.json()
if not data.get("success"):
return f"Scrape failed: {data.get('error', 'unknown error')}"
result = data.get("data", {})
markdown = (result.get("markdown") or "").strip()
if not markdown:
return f"No content extracted from {url}"
title = (result.get("metadata", {}).get("title") or "").strip()
header = f"Page: {title}\nURL: {url}\n\n" if title else f"URL: {url}\n\n"
if len(markdown) > max_chars:
markdown = markdown[:max_chars] + "\n\n[content truncated]"
return header + markdown
if __name__ == "__main__":
mcp.run(transport="stdio")