feat: default to qwen2.5-vl-3b and cap tool schema length constraints
Rebuild & Deploy / rebuild (push) Successful in 54s
Rebuild & Deploy / rebuild (push) Successful in 54s
- Default LLM_MODEL to qwen2.5-vl-3b across agent, task worker, tests, compose, workflow, and docs - Cap min/max length-style constraints in tool schemas to 1000 before LLM requests; llama.cpp rejects grammar repetitions >= 2000 and the memory MCP's update_note body maxLength (200000) broke qwen2.5-vl-3b with 'Failed to initialize samplers: failed to parse grammar'
This commit is contained in:
1 parent
a66f09eb28
commit
a4ae96f01b
7 files changed
+39
-8
No files matched your search
+32
-1
@@ -45,7 +45,7 @@ LLM_BASE_URL = (
|
||||
LLM_MODEL = (
|
||||
os.environ.get("LLM_MODEL")
|
||||
or os.environ.get("GEMMA_MODEL")
|
||||
or "gemma-4-e4b"
|
||||
or "qwen2.5-vl-3b"
|
||||
)
|
||||
LLM_API_KEY = (
|
||||
os.environ.get("LLM_API_KEY")
|
||||
@@ -216,6 +216,28 @@ except ImportError:
|
||||
|
||||
# ── OpenAI-compatible LLM with reasoning_content fallback ───────────────────
|
||||
|
||||
# llama.cpp derives a GBNF grammar from tool schemas to constrain the model's
|
||||
# tool-call output. Its parser rejects repetition counts >= 2000 (e.g. a string
|
||||
# with maxLength 200000 emits "char{0,200000}" which fails to parse), so cap
|
||||
# the length-style constraints the voice pipeline exposes. qwen2.5-vl-3b hits
|
||||
# this; gemma happens to tolerate the same schemas.
|
||||
_LLM_SCHEMA_LENGTH_CEILING = 1000
|
||||
_LLM_SCHEMA_LENGTH_KEYS = ("minLength", "maxLength", "minItems", "maxItems")
|
||||
|
||||
|
||||
def _sanitize_tool_schema(node: object) -> None:
|
||||
"""Cap length-style constraints in a JSON-schema dict (mutates in place)."""
|
||||
if isinstance(node, dict):
|
||||
for key in _LLM_SCHEMA_LENGTH_KEYS:
|
||||
value = node.get(key)
|
||||
if isinstance(value, (int, float)) and value > _LLM_SCHEMA_LENGTH_CEILING:
|
||||
node[key] = _LLM_SCHEMA_LENGTH_CEILING
|
||||
for value in node.values():
|
||||
_sanitize_tool_schema(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_sanitize_tool_schema(item)
|
||||
|
||||
|
||||
class _ReasoningFallbackWrapper:
|
||||
"""Wraps an LLMStream to fall back to ``reasoning_content`` when the model
|
||||
@@ -352,6 +374,15 @@ class HopeLLM(openai.LLM):
|
||||
idle_for = now - self._last_call_time
|
||||
self._last_call_time = now
|
||||
|
||||
# Cap length-style constraints on tool schemas. llama.cpp builds a GBNF
|
||||
# grammar from these and rejects repetitions >= 2000 (qwen2.5-vl-3b
|
||||
# fails on schemas like the memory MCP's update_note body maxLength).
|
||||
for tool in tools or []:
|
||||
info = getattr(tool, "info", None)
|
||||
raw_schema = getattr(info, "raw_schema", None)
|
||||
if isinstance(raw_schema, dict):
|
||||
_sanitize_tool_schema(raw_schema)
|
||||
|
||||
# Compact if context is large, OR if we've been idle for 5+ minutes
|
||||
# (user came back after a break — summarize the old conversation).
|
||||
ctx_len = len(chat_ctx.items)
|
||||
|
||||
Reference in new issue
Block a user