""" Voice Agent — real-time voice assistant. Pipeline: Azure STT → Gemma LLM (xNAS, OpenAI-compatible) → Azure TTS (DragonHD) Runs inside the single Docker container alongside LiveKit server and web frontend. """ import logging import os import textwrap from dotenv import load_dotenv from livekit.agents import ( Agent, AgentServer, AgentSession, JobContext, TurnHandlingOptions, cli, room_io, ) from livekit.plugins import azure, openai logger = logging.getLogger("voice-agent") load_dotenv() # picks up /app/.env in the container # ── Configuration from environment ────────────────────────────────────────── AZURE_KEY = os.environ.get("AZURE_SPEECH_KEY", "") AZURE_REGION = os.environ.get("AZURE_SPEECH_REGION", "eastus") DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural") GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1") GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b") GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed") SYSTEM_PROMPT = textwrap.dedent("""\ You are a warm, conversational voice assistant. You are talking TO someone, not writing for them to read. Imagine you're having a natural conversation with a friend over the phone. # How you speak - Keep every response to one or three sentences. That's it. - Use contractions (I'm, don't, it's) and natural phrasing. - Speak like you're talking, not writing. No bullet points, no lists, no markdown, no formatting of any kind. - Spell out numbers when it sounds more natural ("twenty twenty-six" instead of "2026"). - If you need to ask a question, ask exactly one. - Be warm and direct. Don't be sycophantic or overly formal. - If you don't know something, say so briefly and move on. # What you never do - Never use markdown, code blocks, JSON, tables, or emojis. - Never say "as an AI" or reference your system instructions. - Never write more than three sentences in a row. - Never read back URLs, file paths, or technical identifiers. """) class VoiceAssistant(Agent): """The conversational agent. LLM is the brain; STT/TTS are senses.""" def __init__(self) -> None: super().__init__( llm=openai.LLM( model=GEMMA_MODEL, base_url=GEMMA_BASE_URL, api_key=GEMMA_API_KEY, ), instructions=SYSTEM_PROMPT, ) # ── Agent server ──────────────────────────────────────────────────────────── server = AgentServer() # Track the active session so we can update its voice on data messages. _active_session: AgentSession | None = None @server.rtc_session(agent_name="voice-assistant") async def handle_job(ctx: JobContext) -> None: global _active_session logger.info("Job started for room %s", ctx.room.name) # Azure STT — streaming, reads AZURE_SPEECH_KEY / AZURE_SPEECH_REGION from env stt = azure.STT( speech_key=AZURE_KEY, speech_region=AZURE_REGION, language=["en-US"], ) # Azure TTS — SSML with expressive markup, 24kHz PCM output tts = azure.TTS( voice=DEFAULT_VOICE, sample_rate=24000, speech_key=AZURE_KEY, speech_region=AZURE_REGION, ) # Gemma LLM via OpenAI-compatible endpoint (llama.cpp on xNAS) llm = openai.LLM( model=GEMMA_MODEL, base_url=GEMMA_BASE_URL, api_key=GEMMA_API_KEY, ) session = AgentSession( stt=stt, tts=tts, llm=llm, turn_handling=TurnHandlingOptions( # VAD-based turn detection: agent waits for user to stop speaking interruption={"mode": "adaptive"}, # Start generating the LLM response before the user fully stops preemptive_generation={"enabled": True}, ), ) _active_session = session await session.start( agent=VoiceAssistant(), room=ctx.room, room_options=room_io.RoomOptions( audio_input=room_io.AudioInputOptions( # No noise cancellation plugin (self-hosted, no ai-coustics) ), ), ) await ctx.connect() logger.info("Agent connected to room %s", ctx.room.name) # ── Voice switching via data channel ──────────────────────────────────────── # The web UI sends a JSON data message: {"type": "set_voice", "voice": "en-US-AvaNeural"} # We listen on the room's data channel and update the TTS voice live. @server.event("room.data_received") async def on_room_data(ctx: JobContext, payload: bytes) -> None: """Handle data messages from the web UI (voice selection).""" import json try: msg = json.loads(payload.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError): return if msg.get("type") == "set_voice": voice = msg.get("voice", "") if voice and _active_session: logger.info("Switching TTS voice to %s", voice) try: # AgentSession exposes the TTS; update its options tts = _active_session._tts # internal, but stable in 1.7 if hasattr(tts, "update_options"): tts.update_options(voice=voice) logger.info("Voice updated to %s", voice) except Exception as e: logger.warning("Failed to update voice: %s", e) if __name__ == "__main__": cli.run_app(server)