feat: web access via MCP (Firecrawl search/scrape) + mcp server attach support

- agent/web_mcp.py: stdio MCP server exposing web_search and web_scrape,
  backed by the self-hosted Firecrawl stack on xNAS (no API key needed)
- agent.py: Agent now attaches mcp_servers built from config; EXTRA_MCP_SERVERS
  env var allows adding arbitrary HTTP/SSE MCP servers as JSON
- Dockerfile: installs livekit-agents[mcp], copies web_mcp.py
- .env.example: WEB_MCP_ENABLED, FIRECRAWL_BASE, EXTRA_MCP_SERVERS documented

web/app.js

@@ -2,8 +2,10 @@
 const { Room, RoomEvent } = LivekitClient;
 
 // ── Config ──────────────────────────────────────────────────────────────────
-// LiveKit server runs on the same host. When served over HTTPS, use wss://.
-const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.hostname}:7880`;
+// LiveKit is reached through the same origin as the page (nginx proxies
+// /livekit -> localhost:7880). Keeps a single HTTPS port and avoids mixed
+// content when the UI is served over HTTPS.
+const LIVEKIT_URL = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/livekit`;
 const ROOM_NAME = "voice-room";
 const AGENT_NAME = "voice-assistant";
 
@@ -35,7 +37,102 @@ if (savedVoice) {
     voiceSelect.value = savedVoice;
 }
 
-// ── Start ───────────────────────────────────────────────────────────────────
+// ── Agent detection ─────────────────────────────────────────────────────────
+// The agent may already be in the room before we join (it persists between
+// page loads), so scan remoteParticipants after connect AND watch for new
+// joins — ParticipantConnected does not fire for pre-existing participants.
+function isAgent(participant) {
+    // kind is the numeric proto enum; ParticipantKind.AGENT === 4.
+    // Prefer the SDK's own getter where available.
+    if (participant.isAgent !== undefined && participant.isAgent !== null) {
+        return participant.isAgent;
+    }
+    return participant.identity === AGENT_NAME
+        || participant.kind === LivekitClient.ParticipantKind.AGENT;
+}
+
+function findAgent(rm) {
+    if (!rm) return null;
+    for (const p of rm.remoteParticipants.values()) {
+        if (isAgent(p)) return p;
+    }
+    return null;
+}
+
+function markAgentFound(rm, participant) {
+    agentParticipant = participant;
+    statusEl.textContent = "Agent connected — speak now";
+    // Send current voice selection to the agent
+    sendVoiceMessage(voiceSelect.value, rm);
+}
+
+function createRoom(token) {
+    const newRoom = new Room({
+        adaptiveStream: true,
+        dynacast: true,
+    });
+
+    // Listen for the agent joining after us
+    newRoom.on(RoomEvent.ParticipantConnected, (participant) => {
+        if (isAgent(participant)) markAgentFound(newRoom, participant);
+    });
+
+    // Listen for transcripts from the agent
+    newRoom.on(RoomEvent.DataReceived, (payload) => {
+        try {
+            const msg = JSON.parse(payload.decodeToString());
+            if (msg.type === "transcript") {
+                addMessage(msg.role || "agent", msg.text);
+            }
+        } catch (e) {
+            // Ignore non-JSON data
+        }
+    });
+
+    // Guard with `=== room` so a stale (retry-discarded) connection cannot
+    // clobber UI state of the active one.
+    newRoom.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
+        if (participant === newRoom.localParticipant && newRoom === room) {
+            statusEl.textContent = `Connected (${quality} quality)`;
+        }
+    });
+
+    newRoom.on(RoomEvent.Disconnected, () => {
+        if (newRoom !== room) return;
+        statusEl.textContent = "Disconnected";
+        startBtn.disabled = false;
+        stopBtn.disabled = true;
+        agentParticipant = null;
+    });
+
+    return newRoom.connect(LIVEKIT_URL, token, {
+        autoSubscribe: true,
+    }).then(() => {
+        // Pick up an agent that was already in the room before we joined
+        const existing = findAgent(newRoom);
+        if (existing) markAgentFound(newRoom, existing);
+        return newRoom;
+    });
+}
+
+function waitForAgent(timeoutMs) {
+    return new Promise((resolve) => {
+        if (findAgent(room)) return resolve(true);
+        const deadline = Date.now() + timeoutMs;
+        const timer = setInterval(() => {
+            const found = findAgent(room);
+            if (found) {
+                if (!agentParticipant) markAgentFound(room, found);
+                clearInterval(timer);
+                resolve(true);
+            } else if (Date.now() > deadline) {
+                clearInterval(timer);
+                resolve(false);
+            }
+        }, 250);
+    });
+}
+
 startBtn.addEventListener("click", async () => {
     try {
         statusEl.textContent = "Connecting...";
@@ -43,57 +140,36 @@ startBtn.addEventListener("click", async () => {
         // Get a signed access token from the token endpoint
         const token = await fetchToken(ROOM_NAME);
 
-        room = new Room({
-            adaptiveStream: true,
-            dynacast: true,
-        });
+        room = await createRoom(token);
 
-        // Listen for agent participant joining
-        room.on(RoomEvent.ParticipantConnected, (participant) => {
-            if (participant.identity === AGENT_NAME || participant.kind === "agent") {
-                agentParticipant = participant;
-                statusEl.textContent = "Agent connected — speak now";
-
-                // Send current voice selection to the agent
-                const voice = voiceSelect.value;
-                sendVoiceMessage(voice);
+        // The agent is dispatched when we join. If it misses the dispatch
+        // (e.g. the server just started and the worker wasn't ready yet),
+        // reconnect once to trigger a fresh dispatch.
+        statusEl.textContent = "Waiting for assistant...";
+        if (!(await waitForAgent(10000))) {
+            statusEl.textContent = "Assistant not ready, retrying...";
+            const stale = room;
+            room = null;          // detach first so its Disconnected handler
+            agentParticipant = null;  // cannot clobber the UI
+            await stale.disconnect();
+            room = await createRoom(token);
+            if (!(await waitForAgent(15000))) {
+                throw new Error("Assistant did not join — reload the page and try again");
             }
-        });
+        }
 
-        // Listen for transcripts from the agent
-        room.on(RoomEvent.DataReceived, (payload, participant) => {
-            try {
-                const msg = JSON.parse(payload.decodeToString());
-                if (msg.type === "transcript") {
-                    addMessage(msg.role || "agent", msg.text);
-                }
-            } catch (e) {
-                // Ignore non-JSON data
-            }
-        });
-
-        room.on(RoomEvent.ConnectionQualityChanged, (participant, quality) => {
-            if (participant === room.localParticipant) {
-                statusEl.textContent = `Connected (${quality} quality)`;
-            }
-        });
-
-        room.on(RoomEvent.Disconnected, () => {
-            statusEl.textContent = "Disconnected";
-            startBtn.disabled = false;
-            stopBtn.disabled = true;
-            agentParticipant = null;
-        });
-
-        await room.connect(LIVEKIT_URL, {
-            accessToken: token,
-            autoSubscribe: true,
-        });
+        statusEl.textContent = "Requesting microphone...";
+        await room.localParticipant.setMicrophoneEnabled(true);
 
         statusEl.textContent = "Connected — speak now";
         startBtn.disabled = true;
         stopBtn.disabled = false;
 
+        // Live mic level readout: if this stays at 0 while you talk, the
+        // phone is not capturing audio (iOS quirk), and the problem is on
+        // the device side, not the server.
+        startMicMeter();
+
     } catch (err) {
         console.error("Connection failed:", err);
         statusEl.textContent = `Error: ${err.message}`;
@@ -125,15 +201,48 @@ async function fetchToken(roomName) {
 }
 
 // ── Helper: send voice selection to the agent via data channel ─────────────
-function sendVoiceMessage(voice) {
-    if (!room || !room.localParticipant) return;
+function sendVoiceMessage(voice, targetRoom) {
+    const r = targetRoom || room;
+    if (!r || !r.localParticipant) return;
     const payload = new TextEncoder().encode(JSON.stringify({ type: "set_voice", voice }));
-    room.localParticipant.publishData(payload, {
+    r.localParticipant.publishData(payload, {
         topic: "voice-control",
         reliable: true,
     });
 }
 
+// ── Helper: live mic level readout (diagnostic) ─────────────────────────────
+function startMicMeter() {
+    const el = document.getElementById("micLevel");
+    if (!el) return;
+    let audioCtx = null;
+    let analyser = null;
+    try {
+        const pub = room.localParticipant.getTrackPublication(LivekitClient.Track.Source.Microphone);
+        const mst = pub && pub.track && pub.track.mediaStreamTrack;
+        if (mst) {
+            audioCtx = new AudioContext();
+            analyser = audioCtx.createAnalyser();
+            analyser.fftSize = 512;
+            audioCtx.createMediaStreamSource(new MediaStream([mst])).connect(analyser);
+        }
+    } catch (e) { /* fall back to SDK audioLevel below */ }
+
+    const buf = analyser ? new Float32Array(analyser.fftSize) : null;
+    window.micMeterTimer = setInterval(() => {
+        if (!room || !room.localParticipant) { clearInterval(window.micMeterTimer); return; }
+        let level = 0;
+        if (analyser) {
+            analyser.getFloatTimeDomainData(buf);
+            for (let i = 0; i < buf.length; i++) level = Math.max(level, Math.abs(buf[i]));
+        } else {
+            level = room.localParticipant.audioLevel;
+        }
+        el.textContent = `mic: ${Math.round(level * 100)}%`;
+        el.style.color = level > 0.02 ? "#4ade80" : "#94a3b8";
+    }, 300);
+}
+
 // ── Helper: add a message to the transcript ─────────────────────────────────
 function addMessage(role, text) {
     const div = document.createElement("div");

web/index.html

@@ -37,6 +37,7 @@
         </div>
 
         <div class="status" id="status">Disconnected</div>
+        <div class="status" id="micLevel" style="font-size: 0.8em;"></div>
 
         <div class="transcript" id="transcript">
             <div class="transcript-header">Conversation</div>
@@ -44,7 +45,7 @@
         </div>
     </div>
 
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/index.umd.min.js"></script>
+    <script src="livekit-client.umd.js"></script>
     <script src="app.js"></script>
 </body>
 </html>

web/livekit-client.umd.js

@@ -0,0 +1,2 @@
+!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports):"function"==typeof define&&define.amd?define(["exports"],t):t((e="undefined"!=typeof globalThis?globalThis:e||self).LivekitClient={})}(this,(function(e){"use strict";function t(e,t){return t.forEach((function(t){t&&"string"!=typeof t&&!Array.isArray(t)&&Object.keys(t).forEach((function(n){if("default"!==n&&!(n in e)){var i=Object.getOwnPropertyDescriptor(t,n);Object.defineProperty(e,n,i.get?i:{enumerable:!0,get:function(){return t[n]}})}}))})),Object.freeze(e)}var n=Object.defineProperty,i=(e,t,i)=>((e,t,i)=>t in e?n(e,t,{enumerable:!0,configurable:!0,writable:!0,value:i}):e[t]=i)(e,"symbol"!=typeof t?t+"":t,i);class s{constructor(){i(this,"_locking"),i(this,"_locks"),this._locking=Promise.resolve(),this._locks=0}isLocked(){return this._locks>0}lock(){let e;this._locks+=1;const t=new Promise((t=>e=()=>{this._locks-=1,t()})),n=this._locking.then((()=>e));return this._locking=this._locking.then((()=>t)),n}}function o(e,t){if(!e)throw new Error(t)}const r=34028234663852886e22,a=-34028234663852886e22,c=4294967295,d=2147483647,l=-2147483648;function u(e){if("number"!=typeof e)throw new Error("invalid int 32: "+typeof e);if(!Number.isInteger(e)||e>d||e<l)throw new Error("invalid int 32: "+e)}function h(e){if("number"!=typeof e)throw new Error("invalid uint 32: "+typeof e);if(!Number.isInteger(e)||e>c||e<0)throw new Error("invalid uint 32: "+e)}function p(e){if("number"!=typeof e)throw new Error("invalid float 32: "+typeof e);if(Number.isFinite(e)&&(e>r||e<a))throw new Error("invalid float 32: "+e)}const m=Symbol("@bufbuild/protobuf/enum-type");function g(e){const t=e[m];return o(t,"missing enum type on enum object"),t}function v(e,t,n,i){e[m]=f(t,n.map((t=>({no:t.no,name:t.name,localName:e[t.no]}))))}function f(e,t,n){const i=Object.create(null),s=Object.create(null),o=[];for(const e of t){const t=b(e);o.push(t),i[e.name]=t,s[e.no]=t}return{typeName:e,values:o,findName:e=>i[e],findNumber:e=>s[e]}}function k(e,t,n){const i={};for(const e of t){const t=b(e);i[t.localName]=t.no,i[t.no]=t.localName}return v(i,e,t),i}function b(e){return"localName"in e?e:Object.assign(Object.assign({},e),{localName:e.name})}class y{equals(e){return this.getType().runtime.util.equals(this.getType(),this,e)}clone(){return this.getType().runtime.util.clone(this)}fromBinary(e,t){const n=this.getType().runtime.bin,i=n.makeReadOptions(t);return n.readMessage(this,i.readerFactory(e),e.byteLength,i),this}fromJson(e,t){const n=this.getType(),i=n.runtime.json,s=i.makeReadOptions(t);return i.readMessage(n,e,s,this),this}fromJsonString(e,t){let n;try{n=JSON.parse(e)}catch(e){throw new Error("cannot decode ".concat(this.getType().typeName," from JSON: ").concat(e instanceof Error?e.message:String(e)))}return this.fromJson(n,t)}toBinary(e){const t=this.getType().runtime.bin,n=t.makeWriteOptions(e),i=n.writerFactory();return t.writeMessage(this,i,n),i.finish()}toJson(e){const t=this.getType().runtime.json,n=t.makeWriteOptions(e);return t.writeMessage(this,n)}toJsonString(e){var t;const n=this.toJson(e);return JSON.stringify(n,null,null!==(t=null==e?void 0:e.prettySpaces)&&void 0!==t?t:0)}toJSON(){return this.toJson({emitDefaultValues:!0})}getType(){return Object.getPrototypeOf(this).constructor}}function T(){let e=0,t=0;for(let n=0;n<28;n+=7){let i=this.buf[this.pos++];if(e|=(127&i)<<n,0==(128&i))return this.assertBounds(),[e,t]}let n=this.buf[this.pos++];if(e|=(15&n)<<28,t=(112&n)>>4,0==(128&n))return this.assertBounds(),[e,t];for(let n=3;n<=31;n+=7){let i=this.buf[this.pos++];if(t|=(127&i)<<n,0==(128&i))return this.assertBounds(),[e,t]}throw new Error("invalid varint")}function C(e,t,n){for(let i=0;i<28;i+=7){const s=e>>>i,o=!(s>>>7==0&&0==t),r=255&(o?128|s:s);if(n.push(r),!o)return}const i=e>>>28&15|(7&t)<<4,s=!(t>>3==0);if(n.push(255&(s?128|i:i)),s){for(let e=3;e<31;e+=7){const i=t>>>e,s=!(i>>>7==0),o=255&(s?128|i:i);if(n.push(o),!s)return}n.push(t>>>31&1)}}const S=4294967296;function E(e){const t="-"===e[0];t&&(e=e.slice(1));const n=1e6;let i=0,s=0;function o(t,o){const r=Number(e.slice(t,o));s*=n,i=i*n+r,i>=S&&(s+=i/S|0,i%=S)}return o(-24,-18),o(-18,-12),o(-12,-6),o(-6),t?R(i,s):P(i,s)}function w(e,t){if(({lo:e,hi:t}=function(e,t){return{lo:e>>>0,hi:t>>>0}}(e,t)),t<=2097151)return String(S*t+e);const n=16777215&(e>>>24|t<<8),i=t>>16&65535;let s=(16777215&e)+6777216*n+6710656*i,o=n+8147497*i,r=2*i;const a=1e7;return s>=a&&(o+=Math.floor(s/a),s%=a),o>=a&&(r+=Math.floor(o/a),o%=a),r.toString()+I(o)+I(s)}function P(e,t){return{lo:0|e,hi:0|t}}function R(e,t){return t=~t,e?e=1+~e:t+=1,P(e,t)}const I=e=>{const t=String(e);return"0000000".slice(t.length)+t};function O(e,t){if(e>=0){for(;e>127;)t.push(127&e|128),e>>>=7;t.push(e)}else{for(let n=0;n<9;n++)t.push(127&e|128),e>>=7;t.push(1)}}function D(){let e=this.buf[this.pos++],t=127&e;if(0==(128&e))return this.assertBounds(),t;if(e=this.buf[this.pos++],t|=(127&e)<<7,0==(128&e))return this.assertBounds(),t;if(e=this.buf[thisLine truncated
+//# sourceMappingURL=livekit-client.umd.js.map

web/token_server.py

@@ -3,6 +3,11 @@
 
 Serves POST /token -> { "token": "<signed JWT>" }
 Signs a LiveKit access token with HS256 using the API key/secret from env.
+
+The token carries a roomConfig claim requesting the voice-assistant agent, so
+LiveKit dispatches the agent when the participant joins. Without this, the
+auto-created room would get no agent at all.
+
 Runs as a separate supervisord process; nginx proxies /token to it.
 """
 import base64
@@ -17,24 +22,32 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 
 API_KEY = os.environ.get("LIVEKIT_API_KEY", "devkey")
 API_SECRET = os.environ.get("LIVEKIT_API_SECRET", "devsecret")
+AGENT_NAME = os.environ.get("VOICE_AGENT_NAME", "voice-assistant")
 
 
 def b64url(data: bytes) -> str:
     return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
 
 
-def make_token(room_name: str) -> str:
+def make_token(room_name: str, identity: str) -> str:
     now = int(time.time())
     header = {"alg": "HS256", "typ": "JWT"}
     payload = {
         "iss": API_KEY,
-        "sub": room_name,
+        "sub": identity,
         "nbf": now - 5,
         "exp": now + 3600,
         "jti": str(uuid.uuid4()),
-        "video_grants": {
-            "room_join": True,
+        "identity": identity,
+        "video": {
+            "roomJoin": True,
             "room": room_name,
+            "canPublish": True,
+            "canSubscribe": True,
+            "canPublishData": True,
+        },
+        "roomConfig": {
+            "agents": [{"agentName": AGENT_NAME}],
         },
     }
     h = b64url(json.dumps(header).encode())
@@ -54,8 +67,9 @@ class Handler(BaseHTTPRequestHandler):
         length = int(self.headers.get("Content-Length", 0))
         body = json.loads(self.rfile.read(length) or b"{}")
         room_name = body.get("room", "voice-room")
+        identity = body.get("identity") or f"user-{uuid.uuid4().hex[:8]}"
 
-        token = make_token(room_name)
+        token = make_token(room_name, identity)
         resp = json.dumps({"token": token}).encode()
         self.send_response(200)
         self.send_header("Content-Type", "application/json")