feat: default to qwen2.5-vl-3b and cap tool schema length constraints
Rebuild & Deploy / rebuild (push) Successful in 54s

- Default LLM_MODEL to qwen2.5-vl-3b across agent, task worker, tests,
  compose, workflow, and docs
- Cap min/max length-style constraints in tool schemas to 1000 before
  LLM requests; llama.cpp rejects grammar repetitions >= 2000 and the
  memory MCP's update_note body maxLength (200000) broke qwen2.5-vl-3b
  with 'Failed to initialize samplers: failed to parse grammar'
This commit is contained in:
snowspeeder committed 2026-08-29 12:46:20 -04:00
1 parent a66f09eb28
commit a4ae96f01b
7 files changed
+39 -8

No files matched your search

+32 -1
View File
@@ -45,7 +45,7 @@ LLM_BASE_URL = (
LLM_MODEL = (
os.environ.get("LLM_MODEL")
or os.environ.get("GEMMA_MODEL")
or "gemma-4-e4b"
or "qwen2.5-vl-3b"
)
LLM_API_KEY = (
os.environ.get("LLM_API_KEY")
@@ -216,6 +216,28 @@ except ImportError:
# ── OpenAI-compatible LLM with reasoning_content fallback ───────────────────
# llama.cpp derives a GBNF grammar from tool schemas to constrain the model's
# tool-call output. Its parser rejects repetition counts >= 2000 (e.g. a string
# with maxLength 200000 emits "char{0,200000}" which fails to parse), so cap
# the length-style constraints the voice pipeline exposes. qwen2.5-vl-3b hits
# this; gemma happens to tolerate the same schemas.
_LLM_SCHEMA_LENGTH_CEILING = 1000
_LLM_SCHEMA_LENGTH_KEYS = ("minLength", "maxLength", "minItems", "maxItems")
def _sanitize_tool_schema(node: object) -> None:
"""Cap length-style constraints in a JSON-schema dict (mutates in place)."""
if isinstance(node, dict):
for key in _LLM_SCHEMA_LENGTH_KEYS:
value = node.get(key)
if isinstance(value, (int, float)) and value > _LLM_SCHEMA_LENGTH_CEILING:
node[key] = _LLM_SCHEMA_LENGTH_CEILING
for value in node.values():
_sanitize_tool_schema(value)
elif isinstance(node, list):
for item in node:
_sanitize_tool_schema(item)
class _ReasoningFallbackWrapper:
"""Wraps an LLMStream to fall back to ``reasoning_content`` when the model
@@ -352,6 +374,15 @@ class HopeLLM(openai.LLM):
idle_for = now - self._last_call_time
self._last_call_time = now
# Cap length-style constraints on tool schemas. llama.cpp builds a GBNF
# grammar from these and rejects repetitions >= 2000 (qwen2.5-vl-3b
# fails on schemas like the memory MCP's update_note body maxLength).
for tool in tools or []:
info = getattr(tool, "info", None)
raw_schema = getattr(info, "raw_schema", None)
if isinstance(raw_schema, dict):
_sanitize_tool_schema(raw_schema)
# Compact if context is large, OR if we've been idle for 5+ minutes
# (user came back after a break — summarize the old conversation).
ctx_len = len(chat_ctx.items)