From a4ae96f01bce47f82f26337b17d01564274a631b Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sat, 29 Aug 2026 12:46:20 -0400 Subject: [PATCH] feat: default to qwen2.5-vl-3b and cap tool schema length constraints - Default LLM_MODEL to qwen2.5-vl-3b across agent, task worker, tests, compose, workflow, and docs - Cap min/max length-style constraints in tool schemas to 1000 before LLM requests; llama.cpp rejects grammar repetitions >= 2000 and the memory MCP's update_note body maxLength (200000) broke qwen2.5-vl-3b with 'Failed to initialize samplers: failed to parse grammar' --- .env.example | 2 +- .gitea/workflows/rebuild.yml | 2 +- AGENTS.md | 4 ++-- agent/agent.py | 33 ++++++++++++++++++++++++++++++++- agent/task_worker.py | 2 +- docker-compose.yml | 2 +- tests/conftest.py | 2 +- 7 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 86a22cf..2879ede 100644 --- a/.env.example +++ b/.env.example @@ -8,7 +8,7 @@ AZURE_TTS_VOICE=en-US-Ava:DragonHDOmniLatestNeural # LLM (OpenWebUI on xNAS, OpenAI-compatible; also serves the local llama.cpp # models. The model can be switched live from the web UI settings sheet.) LLM_BASE_URL=http://192.168.86.2:3005/api -LLM_MODEL=gemma-4-e4b +LLM_MODEL=qwen2.5-vl-3b LLM_API_KEY=sk-your-openwebui-api-key # LiveKit (generated in livekit.yaml, referenced here for the agent) diff --git a/.gitea/workflows/rebuild.yml b/.gitea/workflows/rebuild.yml index 31a2d56..8765bd1 100644 --- a/.gitea/workflows/rebuild.yml +++ b/.gitea/workflows/rebuild.yml @@ -41,7 +41,7 @@ jobs: -e AZURE_SPEECH_REGION=eastus \ -e AZURE_TTS_VOICE=en-US-Ava:DragonHDOmniLatestNeural \ -e LLM_BASE_URL=http://192.168.86.2:3005/api \ - -e LLM_MODEL=gemma-4-e4b \ + -e LLM_MODEL=qwen2.5-vl-3b \ -e LLM_API_KEY=sk-4e9259e3bd56472db57fd40adf0fe567 \ -e WEB_MCP_ENABLED=true \ -e FIRECRAWL_BASE=http://192.168.86.2:3002 \ diff --git a/AGENTS.md b/AGENTS.md index 7567c5e..927038a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,7 @@ Key variables: - `AZURE_SPEECH_REGION` — default `eastus` - `AZURE_TTS_VOICE` — default voice (`en-US-Ava:DragonHDOmniLatestNeural`) - `LLM_BASE_URL` — OpenAI-compatible endpoint (default `http://192.168.86.2:3005/api`, OpenWebUI) -- `LLM_MODEL` — default model (default `gemma-4-e4b`; switchable live from the web UI) +- `LLM_MODEL` — default model (default `qwen2.5-vl-3b`; switchable live from the web UI) - `LLM_API_KEY` — OpenWebUI API key (bears `sk-…`) ## Voice Selection @@ -129,7 +129,7 @@ curl -s "https://eastus.tts.speech.microsoft.com/cognitiveservices/v1" \ - **Mic requires HTTPS.** Browsers block getUserMedia outside a secure context. nginx serves the UI on `WEB_PORT` (9178 by default) over HTTPS with a self-signed cert whose SAN includes the detected LAN IP. The cert is persisted in `./certs/` and the LiveKit WS is proxied through nginx at `/livekit/`. - **No CDN dependencies.** livekit-client UMD bundle is vendored into `web/`; LAN devices may have no internet access. - **Transcripts flow over the data channel.** The agent publishes `{type: "transcript", role, text}` JSON on topic "transcript"; the UI renders them. Voice changes flow the other way as `{type: "set_voice", voice}` on topic "voice-control". -- **Models come from OpenWebUI.** The LLM endpoint is OpenWebUI's OpenAI-compatible API at `LLM_BASE_URL` (default `http://192.168.86.2:3005/api`); chat goes to `/api/chat/completions` and the model list comes from `GET {LLM_BASE_URL}/models`. Plain `/v1/*` routes are NOT routed on that instance (they serve the SPA). Pipe models (`opencode_go_messages.*`, `go-chat.*`, `go-responses.*`) currently fail upstream with auth errors; the local llama.cpp/vLLM models (e.g. `gemma-4-e4b`, `qwen3.8-27b`) work. +- **Models come from OpenWebUI.** The LLM endpoint is OpenWebUI's OpenAI-compatible API at `LLM_BASE_URL` (default `http://192.168.86.2:3005/api`); chat goes to `/api/chat/completions` and the model list comes from `GET {LLM_BASE_URL}/models`. Plain `/v1/*` routes are NOT routed on that instance (they serve the SPA). Pipe models (`opencode_go_messages.*`, `go-chat.*`, `go-responses.*`) currently fail upstream with auth errors; the local llama.cpp/vLLM models (e.g. `gemma-4-e4b`, `qwen2.5-vl-3b`, `qwen3.8-27b`) work. - **Reasoning models may hide their answer.** Gemma (and similar) sometimes spends tokens on hidden reasoning before producing content. The agent implements this with `max_completion_tokens=1000` and a `HopeLLM` subclass that wraps the LLM stream, falling back to `reasoning_content` if `content` is empty. - **Azure TTS uses SSML, not JSON.** The REST endpoint requires `Content-Type: application/ssml+xml`. The LiveKit Azure plugin handles this internally. - **Voice changes are live.** The web UI sends a data message to the agent; the agent calls `tts.update_options(voice=...)` without restarting. The agent filters data messages by topic ("voice-control") before processing. diff --git a/agent/agent.py b/agent/agent.py index 7ddd1b4..f8b66f2 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -45,7 +45,7 @@ LLM_BASE_URL = ( LLM_MODEL = ( os.environ.get("LLM_MODEL") or os.environ.get("GEMMA_MODEL") - or "gemma-4-e4b" + or "qwen2.5-vl-3b" ) LLM_API_KEY = ( os.environ.get("LLM_API_KEY") @@ -216,6 +216,28 @@ except ImportError: # ── OpenAI-compatible LLM with reasoning_content fallback ─────────────────── +# llama.cpp derives a GBNF grammar from tool schemas to constrain the model's +# tool-call output. Its parser rejects repetition counts >= 2000 (e.g. a string +# with maxLength 200000 emits "char{0,200000}" which fails to parse), so cap +# the length-style constraints the voice pipeline exposes. qwen2.5-vl-3b hits +# this; gemma happens to tolerate the same schemas. +_LLM_SCHEMA_LENGTH_CEILING = 1000 +_LLM_SCHEMA_LENGTH_KEYS = ("minLength", "maxLength", "minItems", "maxItems") + + +def _sanitize_tool_schema(node: object) -> None: + """Cap length-style constraints in a JSON-schema dict (mutates in place).""" + if isinstance(node, dict): + for key in _LLM_SCHEMA_LENGTH_KEYS: + value = node.get(key) + if isinstance(value, (int, float)) and value > _LLM_SCHEMA_LENGTH_CEILING: + node[key] = _LLM_SCHEMA_LENGTH_CEILING + for value in node.values(): + _sanitize_tool_schema(value) + elif isinstance(node, list): + for item in node: + _sanitize_tool_schema(item) + class _ReasoningFallbackWrapper: """Wraps an LLMStream to fall back to ``reasoning_content`` when the model @@ -352,6 +374,15 @@ class HopeLLM(openai.LLM): idle_for = now - self._last_call_time self._last_call_time = now + # Cap length-style constraints on tool schemas. llama.cpp builds a GBNF + # grammar from these and rejects repetitions >= 2000 (qwen2.5-vl-3b + # fails on schemas like the memory MCP's update_note body maxLength). + for tool in tools or []: + info = getattr(tool, "info", None) + raw_schema = getattr(info, "raw_schema", None) + if isinstance(raw_schema, dict): + _sanitize_tool_schema(raw_schema) + # Compact if context is large, OR if we've been idle for 5+ minutes # (user came back after a break — summarize the old conversation). ctx_len = len(chat_ctx.items) diff --git a/agent/task_worker.py b/agent/task_worker.py index c7d1445..3c3f377 100644 --- a/agent/task_worker.py +++ b/agent/task_worker.py @@ -39,7 +39,7 @@ class TaskWorker: model = ( os.environ.get("LLM_MODEL") or os.environ.get("GEMMA_MODEL") - or "gemma-4-e4b" + or "qwen2.5-vl-3b" ) api_key = ( os.environ.get("LLM_API_KEY") diff --git a/docker-compose.yml b/docker-compose.yml index 4e46127..8e704ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: AZURE_SPEECH_REGION: "${AZURE_SPEECH_REGION:-eastus}" AZURE_TTS_VOICE: "${AZURE_TTS_VOICE:-en-US-Ava:DragonHDOmniLatestNeural}" LLM_BASE_URL: "${LLM_BASE_URL:-http://192.168.86.2:3005/api}" - LLM_MODEL: "${LLM_MODEL:-gemma-4-e4b}" + LLM_MODEL: "${LLM_MODEL:-qwen2.5-vl-3b}" LLM_API_KEY: "${LLM_API_KEY:-not-needed}" WEB_MCP_ENABLED: "${WEB_MCP_ENABLED:-true}" FIRECRAWL_BASE: "${FIRECRAWL_BASE:-http://192.168.86.2:3002}" diff --git a/tests/conftest.py b/tests/conftest.py index e9949b5..61c8e8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ LLM_BASE_URL = ( LLM_MODEL = ( os.environ.get("LLM_MODEL") or os.environ.get("GEMMA_MODEL") - or "gemma-4-e4b" + or "qwen2.5-vl-3b" ) LLM_API_KEY = ( os.environ.get("LLM_API_KEY")