snowspeeder/hope-voice-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: web access via MCP (Firecrawl search/scrape) + mcp server attach support

Browse Source 
- agent/web_mcp.py: stdio MCP server exposing web_search and web_scrape,
  backed by the self-hosted Firecrawl stack on xNAS (no API key needed)
- agent.py: Agent now attaches mcp_servers built from config; EXTRA_MCP_SERVERS
  env var allows adding arbitrary HTTP/SSE MCP servers as JSON
- Dockerfile: installs livekit-agents[mcp], copies web_mcp.py
- .env.example: WEB_MCP_ENABLED, FIRECRAWL_BASE, EXTRA_MCP_SERVERS documented
This commit is contained in:
Shane
2026-08-22 12:29:04 -04:00
parent 6f2b231938
commit 045ddabac7
12 changed files with 452 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/app.js
+158 -49
View File
@@ -2,8 +2,10 @@
const { Room, RoomEvent } = LivekitClient;
// ── Config ──────────────────────────────────────────────────────────────────
// LiveKit server runs on the same host. When served over HTTPS, use wss://.
const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.hostname}:7880`;
// LiveKit is reached through the same origin as the page (nginx proxies
// /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed
// content when the UI is served over HTTPS.
const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`;
const ROOM_NAME = "voice-room";
const AGENT_NAME = "voice-assistant";
@@ -35,7 +37,102 @@ if (savedVoice) {
voiceSelect.value = savedVoice;
}
// ── Start ───────────────────────────────────────────────────────────────────
// ── Agent detection ─────────────────────────────────────────────────────────
// The agent may already be in the room before we join (it persists between
// page loads), so scan remoteParticipants after connect AND watch for new
// joins — ParticipantConnected does not fire for pre-existing participants.
function isAgent(participant) {
// kind is the numeric proto enum; ParticipantKind.AGENT === 4.
// Prefer the SDK's own getter where available.
if (participant.isAgent !== undefined && participant.isAgent !== null) {
return participant.isAgent;
}
return participant.identity === AGENT_NAME
|| participant.kind === LivekitClient.ParticipantKind.AGENT;
}
function findAgent(rm) {
if (!rm) return null;
for (const p of rm.remoteParticipants.values()) {
if (isAgent(p)) return p;
}
return null;
}
function markAgentFound(rm, participant) {
agentParticipant = participant;
statusEl.textContent = "Agent connected — speak now";
// Send current voice selection to the agent
sendVoiceMessage(voiceSelect.value, rm);
}
function createRoom(token) {
const newRoom = new Room({
adaptiveStream: true,
dynacast: true,
});
// Listen for the agent joining after us
newRoom.on(RoomEvent.ParticipantConnected, (participant) => {
if (isAgent(participant)) markAgentFound(newRoom, participant);
});
// Listen for transcripts from the agent
newRoom.on(RoomEvent.DataReceived, (payload) => {
try {
const msg = JSON.parse(payload.decodeToString());
if (msg.type === "transcript") {
addMessage(msg.role || "agent", msg.text);
}
} catch (e) {
// Ignore non-JSON data
}
});
// Guard with `=== room` so a stale (retry-discarded) connection cannot
// clobber UI state of the active one.
newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
if (participant === newRoom.localParticipant && newRoom === room) {
statusEl.textContent = `Connected (${quality} quality)`;
}
});
newRoom.on(RoomEvent.Disconnected, () => {
if (newRoom !== room) return;
statusEl.textContent = "Disconnected";
startBtn.disabled = false;
stopBtn.disabled = true;
agentParticipant = null;
});
return newRoom.connect(LIVEKIT_URL, token, {
autoSubscribe: true,
}).then(() => {
// Pick up an agent that was already in the room before we joined
const existing = findAgent(newRoom);
if (existing) markAgentFound(newRoom, existing);
return newRoom;
});
}
function waitForAgent(timeoutMs) {
return new Promise((resolve) => {
if (findAgent(room)) return resolve(true);
const deadline = Date.now() + timeoutMs;
const timer = setInterval(() => {
const found = findAgent(room);
if (found) {
if (!agentParticipant) markAgentFound(room, found);
clearInterval(timer);
resolve(true);
} else if (Date.now() > deadline) {
clearInterval(timer);
resolve(false);
}
}, 250);
});
}
startBtn.addEventListener("click", async () => {
try {
statusEl.textContent = "Connecting...";
@@ -43,57 +140,36 @@ startBtn.addEventListener("click", async () => {
// Get a signed access token from the token endpoint
const token = await fetchToken(ROOM_NAME);
room = new Room({
adaptiveStream: true,
dynacast: true,
});
room = await createRoom(token);
// Listen for agent participant joining
room.on(RoomEvent.ParticipantConnected, (participant) => {
if (participant.identity === AGENT_NAME || participant.kind === "agent") {
agentParticipant = participant;
statusEl.textContent = "Agent connected — speak now";
// Send current voice selection to the agent
const voice = voiceSelect.value;
sendVoiceMessage(voice);
// The agent is dispatched when we join. If it misses the dispatch
// (e.g. the server just started and the worker wasn't ready yet),
// reconnect once to trigger a fresh dispatch.
statusEl.textContent = "Waiting for assistant...";
if (!(await waitForAgent(10000))) {
statusEl.textContent = "Assistant not ready, retrying...";
const stale = room;
room = null; // detach first so its Disconnected handler
agentParticipant = null; // cannot clobber the UI
await stale.disconnect();
room = await createRoom(token);
if (!(await waitForAgent(15000))) {
throw new Error("Assistant did not join — reload the page and try again");
}
});
}
// Listen for transcripts from the agent
room.on(RoomEvent.DataReceived, (payload, participant) => {
try {
const msg = JSON.parse(payload.decodeToString());
if (msg.type === "transcript") {
addMessage(msg.role || "agent", msg.text);
}
} catch (e) {
// Ignore non-JSON data
}
});
room.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
if (participant === room.localParticipant) {
statusEl.textContent = `Connected (${quality} quality)`;
}
});
room.on(RoomEvent.Disconnected, () => {
statusEl.textContent = "Disconnected";
startBtn.disabled = false;
stopBtn.disabled = true;
agentParticipant = null;
});
await room.connect(LIVEKIT_URL, {
accessToken: token,
autoSubscribe: true,
});
statusEl.textContent = "Requesting microphone...";
await room.localParticipant.setMicrophoneEnabled(true);
statusEl.textContent = "Connected — speak now";
startBtn.disabled = true;
stopBtn.disabled = false;
// Live mic level readout: if this stays at 0 while you talk, the
// phone is not capturing audio (iOS quirk), and the problem is on
// the device side, not the server.
startMicMeter();
} catch (err) {
console.error("Connection failed:", err);
statusEl.textContent = `Error: ${err.message}`;
@@ -125,15 +201,48 @@ async function fetchToken(roomName) {
}
// ── Helper: send voice selection to the agent via data channel ─────────────
function sendVoiceMessage(voice) {
if (!room || !room.localParticipant) return;
function sendVoiceMessage(voice, targetRoom) {
const r = targetRoom || room;
if (!r || !r.localParticipant) return;
const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice }));
room.localParticipant.publishData(payload, {
r.localParticipant.publishData(payload, {
topic: "voice-control",
reliable: true,
});
}
// ── Helper: live mic level readout (diagnostic) ─────────────────────────────
function startMicMeter() {
const el = document.getElementById("micLevel");
if (!el) return;
let audioCtx = null;
let analyser = null;
try {
const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
const mst = pub && pub.track && pub.track.mediaStreamTrack;
if (mst) {
audioCtx = new AudioContext();
analyser = audioCtx.createAnalyser();
analyser.fftSize = 512;
audioCtx.createMediaStreamSource(new MediaStream([mst])).connect(analyser);
}
} catch (e) { /* fall back to SDK audioLevel below */ }
const buf = analyser ? new Float32Array(analyser.fftSize) : null;
window.micMeterTimer = setInterval(() => {
if (!room || !room.localParticipant) { clearInterval(window.micMeterTimer); return; }
let level = 0;
if (analyser) {
analyser.getFloatTimeDomainData(buf);
for (let i = 0; i < buf.length; i++) level = Math.max(level, Math.abs(buf[i]));
} else {
level = room.localParticipant.audioLevel;
}
el.textContent = `mic: ${Math.round(level * 100)}%`;
el.style.color = level > 0.02 ? "#4ade80" : "#94a3b8";
}, 300);
}
// ── Helper: add a message to the transcript ─────────────────────────────────
function addMessage(role, text) {
const div = document.createElement("div");
web/index.html
+2 -1
View File
@@ -37,6 +37,7 @@
</div>
<div class="status" id="status">Disconnected</div>
<div class="status" id="micLevel" style="font-size: 0.8em;"></div>
<div class="transcript" id="transcript">
<div class="transcript-header">Conversation</div>
@@ -44,7 +45,7 @@
</div>
</div>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/index.umd.min.js"></script>
<script src="livekit-client.umd.js"></script>
<script src="app.js"></script>
</body>
</html>
web/livekit-client.umd.js
+2
View File
File diff suppressed because one or more lines are too long
web/token_server.py
+19 -5
View File
@@ -3,6 +3,11 @@
Serves POST /token -> { "token": "<signed JWT>" }
Signs a LiveKit access token with HS256 using the API key/secret from env.
The token carries a roomConfig claim requesting the voice-assistant agent, so
LiveKit dispatches the agent when the participant joins. Without this, the
auto-created room would get no agent at all.
Runs as a separate supervisord process; nginx proxies /token to it.
"""
import base64
@@ -17,24 +22,32 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
API_KEY = os.environ.get("LIVEKIT_API_KEY", "devkey")
API_SECRET = os.environ.get("LIVEKIT_API_SECRET", "devsecret")
AGENT_NAME = os.environ.get("VOICE_AGENT_NAME", "voice-assistant")
def b64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
def make_token(room_name: str) -> str:
def make_token(room_name: str, identity: str) -> str:
now = int(time.time())
header = {"alg": "HS256", "typ": "JWT"}
payload = {
"iss": API_KEY,
"sub": room_name,
"sub": identity,
"nbf": now - 5,
"exp": now + 3600,
"jti": str(uuid.uuid4()),
"video_grants": {
"room_join": True,
"identity": identity,
"video": {
"roomJoin": True,
"room": room_name,
"canPublish": True,
"canSubscribe": True,
"canPublishData": True,
},
"roomConfig": {
"agents": [{"agentName": AGENT_NAME}],
},
}
h = b64url(json.dumps(header).encode())
@@ -54,8 +67,9 @@ class Handler(BaseHTTPRequestHandler):
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length) or b"{}")
room_name = body.get("room", "voice-room")
identity = body.get("identity") or f"user-{uuid.uuid4().hex[:8]}"
token = make_token(room_name)
token = make_token(room_name, identity)
resp = json.dumps({"token": token}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")