diff --git a/QUALIFICATION.md b/QUALIFICATION.md index 2c04c38..7c3e30c 100644 --- a/QUALIFICATION.md +++ b/QUALIFICATION.md @@ -77,6 +77,28 @@ also passes `model_options.reasoning.enabled: false` and forwards content only. assertion races; synchronization fixed, focused settings/large-type/account switch tests passed. No claim of one fully green expanded UI run. +## Account model and thinking settings + +- The authenticated Hope settings sheet loads the Manifest upstream `/v1/models` + catalog through the trusted control plane. Only public IDs/names and application + effort choices are returned; the browser never receives provider credentials. +- `GET` / `PUT /api/settings/model` operate only on the signed-in account. Preferences + persist in a separate SQLite table with account-deletion cascade. Existing accounts + default to `auto` / `none`. Cookie mutations require a trusted Origin. +- The speech bridge resolves authorized room settings at every request, including + existing conversations, and snapshots them for that turn. No interruption/replay. + It sends the chosen model with explicit `provider: custom:hope-upstream`; a bare + model would be ignored by the deployed Hermes API's default policy. +- Thinking choices are requested efforts, not guarantees of model support. Manifest + currently provides no capability metadata; Hermes may clamp levels or omit a + disabled reasoning field where an upstream model cannot represent `none`. +- Reasoning content is still excluded from transcripts/speech. Discovery failures + preserve saved settings and disable changes rather than silently selecting a model. +- Hermes gateway `/model` session overrides have higher precedence than ordinary + OpenAI-compatible request overrides. Hope does not expose that session command. + The deployed configuration inspected has no fallback chain. OpenAI SSE `model` + echoes the request and is not independent proof of the upstream executing model. + ## Operational behavior and boundaries - Each user has separate persistent state, API credential, bridge network, diff --git a/agent/agent.py b/agent/agent.py index 5a1cf7c..d176c95 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -111,9 +111,17 @@ async def entrypoint(ctx: JobContext): await stop_notice() notice_task = spawn(play_notice(text)) + async def model_settings(): + # Fetch at each turn, not just on join/poll: saved account changes apply + # immediately to the next request even in already-connected rooms. + current = await resolve_room(ctx.room.name) + if current["user"]["id"] != user_id: + raise RuntimeError("Account mismatch") + return current["model_settings"] + bridge = HermesLLM(base_url=account["hermes_url"], api_key=account["hermes_api_key"], user_id=user_id, room_id=ctx.room.name, on_progress=progress, on_wait_notice=wait_notice, - on_answer_start=stop_notice) + on_answer_start=stop_notice, get_model_settings=model_settings) tts = azure.TTS(voice=DEFAULT_VOICE, sample_rate=24000, speech_key=os.getenv("AZURE_SPEECH_KEY", ""), speech_region=os.getenv("AZURE_SPEECH_REGION", "eastus")) session = AgentSession(stt=azure.STT(speech_key=os.getenv("AZURE_SPEECH_KEY", ""), diff --git a/agent/hermes_llm.py b/agent/hermes_llm.py index 0cd63c2..45398b4 100644 --- a/agent/hermes_llm.py +++ b/agent/hermes_llm.py @@ -103,6 +103,7 @@ class HermesLLM(lk_llm.LLM): on_progress: ProgressCallback, on_wait_notice: WaitNoticeCallback | None = None, on_answer_start: Callable[[], Awaitable[None]] | None = None, + get_model_settings: Callable[[], Awaitable[dict[str, str]]] | None = None, timeout_seconds: float = 600.0, http_client: httpx.AsyncClient | None = None, ) -> None: @@ -128,6 +129,7 @@ class HermesLLM(lk_llm.LLM): self._on_wait_notice = on_wait_notice self._last_wait_notice: str | None = None self._on_answer_start = on_answer_start + self._get_model_settings = get_model_settings self._timeout_seconds = timeout_seconds self._owns_client = http_client is None self._client = http_client if http_client is not None else httpx.AsyncClient(follow_redirects=False) @@ -219,10 +221,25 @@ class HermesLLMStream(lk_llm.LLMStream): await asyncio.gather(timer, return_exceptions=True) async def _consume(self) -> None: + settings = {"model": "auto", "thinking_level": "none"} + if self._hermes._get_model_settings is not None: + try: + settings = await self._hermes._get_model_settings() + except Exception: + # Never fall back to stale preferences or run after revocation. + raise HermesLLMError("settings_unavailable") from None + model, effort = settings.get("model"), settings.get("thinking_level") + if (not isinstance(model, str) or not 1 <= len(model) <= 200 + or not all(33 <= ord(c) <= 126 for c in model) + or effort not in ("none", "minimal", "low", "medium", "high", "xhigh")): + raise HermesLLMError("invalid_settings") + reasoning = {"enabled": False} if effort == "none" else {"enabled": True, "effort": effort} payload = { - "model": "hermes-agent", "stream": True, + # Explicit provider is required: Hermes otherwise ignores bare model + # requests by default. Neither provider nor API origin is client input. + "model": model, "provider": "custom:hope-upstream", "stream": True, "messages": _messages(self.chat_ctx), - "model_options": {"reasoning": {"enabled": False}}, + "model_options": {"reasoning": reasoning}, } finished = False async with self._hermes._client.stream( diff --git a/control/accounts.py b/control/accounts.py index 9082adc..b2f8f19 100644 --- a/control/accounts.py +++ b/control/accounts.py @@ -14,6 +14,8 @@ from pathlib import Path PUBLIC_FIELDS = ('id', 'username', 'display_name', 'role', 'status', 'container_name', 'error') SLUG = re.compile(r'[a-z][a-z0-9]*(?:-[a-z0-9]+)*\Z') +MODEL = re.compile(r'[!-~]{1,200}\Z') +THINKING_LEVELS = ('none', 'minimal', 'low', 'medium', 'high', 'xhigh') def public_user(user): @@ -74,6 +76,10 @@ class Accounts: error TEXT, password_hash TEXT NOT NULL, hermes_url TEXT, hermes_api_key TEXT, desired TEXT NOT NULL DEFAULT 'ready' ); + CREATE TABLE IF NOT EXISTS account_model_settings ( + user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + model TEXT NOT NULL, thinking_level TEXT NOT NULL + ); CREATE TABLE IF NOT EXISTS sessions ( hash TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, expires_at REAL NOT NULL @@ -105,6 +111,38 @@ class Accounts: row = self.db.execute('SELECT * FROM users WHERE id=?', (user_id,)).fetchone() return dict(row) if row else None + def get_model_settings(self, user_id): + with self.lock: + # The join checks account availability and reads settings in one + # snapshot, including accounts that have not finished provisioning. + row = self.db.execute(''' + SELECT COALESCE(s.model, 'auto') AS model, + COALESCE(s.thinking_level, 'none') AS thinking_level + FROM users u LEFT JOIN account_model_settings s ON s.user_id=u.id + WHERE u.id=? AND u.desired!='deleted' + ''', (user_id,)).fetchone() + if row is None: + raise KeyError(user_id) + return dict(row) + + def set_model_settings(self, user_id, model, thinking_level): + if not isinstance(model, str) or not MODEL.fullmatch(model): + raise ValueError('Model must contain 1–200 printable ASCII characters without whitespace') + if not isinstance(thinking_level, str) or thinking_level not in THINKING_LEVELS: + raise ValueError('Invalid thinking level') + with self.lock, self.db: + # Guard and upsert are one write statement: a delete on another + # connection cannot interleave between checking and storing. + cursor = self.db.execute(''' + INSERT INTO account_model_settings(user_id,model,thinking_level) + SELECT id,?,? FROM users WHERE id=? AND desired!='deleted' + ON CONFLICT(user_id) DO UPDATE SET + model=excluded.model, thinking_level=excluded.thinking_level + ''', (model, thinking_level, user_id)) + if cursor.rowcount != 1: + raise KeyError(user_id) + return {'model': model, 'thinking_level': thinking_level} + def by_username(self, username): with self.lock: row = self.db.execute('SELECT * FROM users WHERE username=?', (username,)).fetchone() diff --git a/control/app.py b/control/app.py index 5d970b9..539eb59 100644 --- a/control/app.py +++ b/control/app.py @@ -206,7 +206,7 @@ class Lifecycle: await asyncio.gather(*tasks, return_exceptions=True) -def create_app(settings=None, provisioner=None): +def create_app(settings=None, provisioner=None, model_catalog=None): settings = settings or Settings() @asynccontextmanager @@ -222,6 +222,9 @@ def create_app(settings=None, provisioner=None): if adapter is None: from .provisioner import Provisioner as HermesProvisioner adapter = HermesProvisioner() + from .model_catalog import ModelCatalog + catalog = model_catalog if model_catalog is not None else ModelCatalog() + app.state.model_catalog = catalog app.state.lifecycle = Lifecycle(store, adapter) # Revalidate even previously-ready containers after control restart. for user in store.users(): @@ -234,6 +237,8 @@ def create_app(settings=None, provisioner=None): loop.cancel() await asyncio.gather(loop, return_exceptions=True) await app.state.lifecycle.stop() + if hasattr(catalog, 'aclose'): + await catalog.aclose() if hasattr(adapter, 'aclose'): await adapter.aclose() finally: @@ -315,6 +320,41 @@ def create_app(settings=None, provisioner=None): async def me(user=Depends(current_user)): return {'user': user} + @app.get('/api/settings/model') + async def model_settings(user=Depends(current_user)): + try: + saved = await asyncio.to_thread(app.state.accounts.get_model_settings, user['id']) + except KeyError: + raise HTTPException(401, 'Account unavailable') from None + try: + models = await app.state.model_catalog.models() + available, message = True, '' + except Exception: + models, available = [], False + message = 'Model list unavailable. Your saved settings are unchanged; try again.' + return {'settings': saved, 'models': models, 'catalog_available': available, + 'message': message, 'thinking_note': 'Thinking levels are requests. Some models may limit or ignore them.'} + + @app.put('/api/settings/model') + async def save_model_settings(request: Request, user=Depends(current_user)): + body = await bounded_json(request) + if set(body) != {'model', 'thinking_level'}: + raise HTTPException(400, 'Only model and thinking_level are accepted') + try: + models = await app.state.model_catalog.models() + except Exception: + raise HTTPException(503, 'Model list unavailable. Settings were not changed; try again.') from None + selected = next((m for m in models if m['id'] == body.get('model')), None) + if selected is None or body.get('thinking_level') not in selected['thinking_levels']: + raise HTTPException(400, 'Choose an available model and thinking level') + try: + saved = await asyncio.to_thread(app.state.accounts.set_model_settings, user['id'], body['model'], body['thinking_level']) + except ValueError: + raise HTTPException(400, 'Invalid model settings') from None + except KeyError: + raise HTTPException(401, 'Account unavailable') from None + return {'settings': saved, 'message': 'Saved for your account. Applies to your next request.'} + @app.get('/api/admin/users') async def list_users(user=Depends(administrator)): return {'users': [public_user(u) for u in await asyncio.to_thread(app.state.accounts.users)]} @@ -366,7 +406,8 @@ def create_app(settings=None, provisioner=None): raise HTTPException(404, 'User not found') if user['status'] != 'ready' or not user['hermes_url'] or not user['hermes_api_key']: raise HTTPException(503, 'User Hermes is not ready') - return {'user': public_user(user), 'hermes_url': user['hermes_url'], 'hermes_api_key': user['hermes_api_key']} + return {'user': public_user(user), 'hermes_url': user['hermes_url'], 'hermes_api_key': user['hermes_api_key'], + 'model_settings': app.state.accounts.get_model_settings(user['id'])} @app.get('/internal/users/{user_id}', dependencies=[Depends(internal_authorized)]) async def internal_user(user_id: str): diff --git a/control/model_catalog.py b/control/model_catalog.py new file mode 100644 index 0000000..16e729d --- /dev/null +++ b/control/model_catalog.py @@ -0,0 +1,63 @@ +"""Trusted upstream discovery. Never return provider URLs, keys or raw metadata.""" +from __future__ import annotations + +import asyncio +import os +import time + +import httpx + + +class CatalogUnavailable(Exception): + pass + + +class ModelCatalog: + def __init__(self, *, base_url=None, api_key=None, client=None): + self.base_url = (base_url or os.getenv('HERMES_UPSTREAM_URL', 'http://192.168.86.5:2099/v1')).rstrip('/') + self.api_key = api_key if api_key is not None else os.getenv('HERMES_UPSTREAM_API_KEY', '') + self.client = client or httpx.AsyncClient(timeout=10, trust_env=False, follow_redirects=False) + self._owns_client = client is None + self._cache = None + self._expires = 0 + self._lock = asyncio.Lock() + + async def aclose(self): + if self._owns_client: + await self.client.aclose() + + async def models(self): + async with self._lock: + if self._cache is not None and time.monotonic() < self._expires: + return self._cache + try: + if not self.api_key: + raise CatalogUnavailable() + async with self.client.stream('GET', self.base_url + '/models', headers={'Authorization': 'Bearer ' + self.api_key}) as response: + response.raise_for_status() + body = bytearray() + async for chunk in response.aiter_bytes(): + body.extend(chunk) + if len(body) > 2 * 1024 * 1024: + raise CatalogUnavailable() + import json + data = json.loads(body).get('data') + if not isinstance(data, list): + raise CatalogUnavailable() + models = [] + seen = set() + for item in data[:1000]: + if not isinstance(item, dict): + continue + model = item.get('id') + if not isinstance(model, str) or not 1 <= len(model) <= 200 or not all(33 <= ord(c) <= 126 for c in model) or model in seen: + continue + seen.add(model) + models.append({'id': model, 'name': model, 'thinking_levels': ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']}) + if not models: + raise CatalogUnavailable() + self._cache = models + self._expires = time.monotonic() + 60 + return models + except (httpx.HTTPError, ValueError, TypeError, AttributeError, CatalogUnavailable): + raise CatalogUnavailable() from None diff --git a/tests/unit/test_hermes_llm.py b/tests/unit/test_hermes_llm.py index fd4ca03..6254282 100644 --- a/tests/unit/test_hermes_llm.py +++ b/tests/unit/test_hermes_llm.py @@ -99,7 +99,7 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(request.headers["x-hermes-session-id"], "stable-room") self.assertEqual(request.headers["x-hermes-session-key"], "hope:alice") self.assertEqual(json.loads(request.content), { - "model": "hermes-agent", "stream": True, + "model": "auto", "provider": "custom:hope-upstream", "stream": True, "messages": [ {"role": "system", "content": "EXACT system prompt\nDo not alter."}, {"role": "user", "content": "hello\nworld"}, @@ -347,6 +347,32 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(result.text, "Done") self.assertEqual(len(self.requests), 1) + async def test_saved_model_and_thinking_refreshed_per_turn(self): + selected = {'model': 'openai/gpt-example', 'thinking_level': 'high'} + async def settings(): + return dict(selected) + bridge = self.bridge(FakeSSE([data({'content': 'Done'}), b'data: [DONE]\n\n']), get_model_settings=settings) + for effort in ('high', 'none', 'low'): + selected['thinking_level'] = effort + await bridge.chat(chat_ctx=self.context(), extra_kwargs={'provider': 'evil', 'model': 'evil'}).collect() + body = json.loads(self.requests[-1].content) + self.assertEqual(body['model'], 'openai/gpt-example') + self.assertEqual(body['provider'], 'custom:hope-upstream') + expected = {'enabled': False} if effort == 'none' else {'enabled': True, 'effort': effort} + self.assertEqual(body['model_options']['reasoning'], expected) + selected['model'] = 'intel-b70-1/Qwen3.8-27b' + await bridge.chat(chat_ctx=self.context()).collect() + self.assertEqual(json.loads(self.requests[-1].content)['model'], selected['model']) + + async def test_settings_failure_does_not_call_hermes_or_leak(self): + async def broken(): + raise RuntimeError('private-key') + with self.assertRaises(HermesLLMError) as caught: + await self.bridge(FakeSSE([]), get_model_settings=broken).chat(chat_ctx=self.context()).collect() + self.assertEqual(caught.exception.code, 'settings_unavailable') + self.assertNotIn('private-key', str(caught.exception)) + self.assertEqual(self.requests, []) + async def test_transport_exception_redacted(self): def broken(request): self.requests.append(request) diff --git a/tests/unit/test_model_preferences_store.py b/tests/unit/test_model_preferences_store.py new file mode 100644 index 0000000..b78b138 --- /dev/null +++ b/tests/unit/test_model_preferences_store.py @@ -0,0 +1,173 @@ +"""Offline account-wide model preferences; no HTTP, Docker, or provider calls.""" +from concurrent.futures import ThreadPoolExecutor +import importlib.util +from pathlib import Path +import sqlite3 +import threading + +import pytest + +# Load only the stdlib store; control.__init__ imports the optional FastAPI app. +_spec = importlib.util.spec_from_file_location( + 'model_preferences_accounts', Path(__file__).resolve().parents[2] / 'control' / 'accounts.py' +) +_accounts = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_accounts) +Accounts = _accounts.Accounts +public_user = _accounts.public_user + + +PASSWORD = 'correct-horse-password' +DEFAULTS = {'model': 'auto', 'thinking_level': 'none'} + + +@pytest.fixture +def accounts(tmp_path): + store = Accounts(str(tmp_path / 'accounts.sqlite3')) + try: + yield store + finally: + store.close() + + +def create_user(store, username='alice'): + return store.create(username, username.title(), 'user', PASSWORD)['id'] + + +def test_defaults_and_saved_state_survive_reopen(tmp_path): + path = str(tmp_path / 'accounts.sqlite3') + store = Accounts(path) + try: + user_id = create_user(store) + assert store.get_model_settings(user_id) == DEFAULTS + assert store.get(user_id)['status'] == 'provisioning' + saved = {'model': 'openai/gpt-5.4', 'thinking_level': 'xhigh'} + assert store.set_model_settings(user_id, **saved) == saved + finally: + store.close() + reopened = Accounts(path) + try: + assert reopened.get_model_settings(user_id) == saved + updated = {'model': 'anthropic/claude-sonnet-4.6', 'thinking_level': 'low'} + reopened.set_model_settings(user_id, **updated) + assert reopened.get_model_settings(user_id) == updated + assert reopened.db.execute('SELECT count(*) FROM account_model_settings').fetchone()[0] == 1 + finally: + reopened.close() + + +def test_existing_database_migrates_with_default_preferences(tmp_path): + path = str(tmp_path / 'legacy.sqlite3') + with sqlite3.connect(path) as legacy: + legacy.executescript(''' + CREATE TABLE users ( + id TEXT PRIMARY KEY, username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, role TEXT NOT NULL, + status TEXT NOT NULL, container_name TEXT NOT NULL, + error TEXT, password_hash TEXT NOT NULL, + hermes_url TEXT, hermes_api_key TEXT, + desired TEXT NOT NULL DEFAULT 'ready' + ); + CREATE TABLE rooms ( + room_name TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at REAL NOT NULL + ); + INSERT INTO users(id,username,display_name,role,status,container_name,password_hash) + VALUES('existing','existing','Existing','user','error','hope-hermes-existing','unused'); + ''') + store = Accounts(path) + try: + assert store.get_model_settings('existing') == DEFAULTS + assert store.get('existing')['status'] == 'error' + assert store.db.execute('SELECT count(*) FROM account_model_settings').fetchone()[0] == 0 + store.set_model_settings('existing', 'openai/gpt-5', 'medium') + assert store.get_model_settings('existing') == {'model': 'openai/gpt-5', 'thinking_level': 'medium'} + finally: + store.close() + + +def test_tenant_isolation_and_public_schema_unchanged(accounts): + alice = create_user(accounts) + bob = create_user(accounts, 'bob') + before = public_user(accounts.get(alice)) + accounts.set_model_settings(alice, 'openai/gpt-5', 'high') + assert accounts.get_model_settings(bob) == DEFAULTS + accounts.set_model_settings(bob, 'local/model:v2', 'minimal') + assert accounts.get_model_settings(alice) == {'model': 'openai/gpt-5', 'thinking_level': 'high'} + assert accounts.get_model_settings(bob) == {'model': 'local/model:v2', 'thinking_level': 'minimal'} + assert public_user(accounts.get(alice)) == before + assert set(before) == {'id', 'username', 'display_name', 'role', 'status', 'container_name', 'error'} + result = accounts.get_model_settings(alice) + result['model'] = 'tampered' + assert accounts.get_model_settings(alice)['model'] == 'openai/gpt-5' + + +def test_missing_deleting_and_deleted_accounts_reject_and_cascade(accounts): + with pytest.raises(KeyError): + accounts.get_model_settings('missing') + with pytest.raises(KeyError): + accounts.set_model_settings('missing', 'auto', 'none') + user_id = create_user(accounts) + accounts.set_model_settings(user_id, 'openai/gpt-5', 'high') + accounts.begin_delete(user_id) + with pytest.raises(KeyError): + accounts.get_model_settings(user_id) + with pytest.raises(KeyError): + accounts.set_model_settings(user_id, 'auto', 'none') + accounts.finish_delete(user_id) + assert accounts.db.execute('SELECT * FROM account_model_settings WHERE user_id=?', (user_id,)).fetchone() is None + with pytest.raises(KeyError): + accounts.get_model_settings(user_id) + with pytest.raises(KeyError): + accounts.set_model_settings(user_id, 'auto', 'none') + assert accounts.db.execute('SELECT count(*) FROM account_model_settings').fetchone()[0] == 0 + + +@pytest.mark.parametrize('model', [None, 123, [], '', 'a' * 201, 'two models', ' auto', 'auto ', + 'auto\n', 'auto\t', 'auto\r', 'a\x00b', 'a\x1fb', 'a\x7fb', 'café', 'a\u00a0b']) +def test_invalid_models_reject_without_changing_saved_state(accounts, model): + user_id = create_user(accounts) + accounts.set_model_settings(user_id, 'valid/model', 'low') + with pytest.raises(ValueError): + accounts.set_model_settings(user_id, model, 'high') + assert accounts.get_model_settings(user_id) == {'model': 'valid/model', 'thinking_level': 'low'} + + +@pytest.mark.parametrize('thinking_level', [None, 1, [], '', 'NONE', 'disabled', 'max', ' high', 'high ', 'high\n']) +def test_invalid_thinking_levels_reject_without_creating_settings(accounts, thinking_level): + user_id = create_user(accounts) + with pytest.raises(ValueError): + accounts.set_model_settings(user_id, 'auto', thinking_level) + assert accounts.get_model_settings(user_id) == DEFAULTS + assert accounts.db.execute('SELECT count(*) FROM account_model_settings').fetchone()[0] == 0 + + +@pytest.mark.parametrize('thinking_level', ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']) +def test_all_thinking_levels_and_model_length_boundary(accounts, thinking_level): + user_id = create_user(accounts) + accounts.set_model_settings(user_id, 'a' * 200, thinking_level) + assert accounts.get_model_settings(user_id) == {'model': 'a' * 200, 'thinking_level': thinking_level} + + +def test_set_is_guarded_against_deletion_on_another_connection(accounts, tmp_path): + user_id = create_user(accounts) + accounts.set_model_settings(user_id, 'original/model', 'none') + other = Accounts(str(tmp_path / 'accounts.sqlite3')) + attempted = threading.Event() + other.db.set_trace_callback(lambda sql: attempted.set() if 'INSERT INTO account_model_settings' in sql else None) + try: + with ThreadPoolExecutor(max_workers=1) as executor: + with accounts.db: + accounts.db.execute('BEGIN IMMEDIATE') + accounts.db.execute("UPDATE users SET desired='deleted',status='deleting' WHERE id=?", (user_id,)) + future = executor.submit(other.set_model_settings, user_id, 'new/model', 'high') + assert attempted.wait(timeout=2), 'Writer did not attempt its guarded statement' + with pytest.raises(KeyError): + future.result(timeout=5) + row = accounts.db.execute('SELECT model,thinking_level FROM account_model_settings WHERE user_id=?', (user_id,)).fetchone() + assert dict(row) == {'model': 'original/model', 'thinking_level': 'none'} + accounts.finish_delete(user_id) + assert accounts.db.execute('SELECT count(*) FROM account_model_settings').fetchone()[0] == 0 + finally: + other.close() diff --git a/tests/unit/test_model_settings_api.py b/tests/unit/test_model_settings_api.py new file mode 100644 index 0000000..e4d5906 --- /dev/null +++ b/tests/unit/test_model_settings_api.py @@ -0,0 +1,105 @@ +"""Offline authenticated model settings/discovery contracts.""" +import json +import pytest +import httpx +from fastapi.testclient import TestClient +from control.app import Settings, create_app +from control.model_catalog import ModelCatalog, CatalogUnavailable + +PASSWORD = 'correct-horse-password' +SECRET = 'internal-secret-' + 'x' * 32 + +class Provisioner: + async def ensure(self, user): + return {'container_name': user['container_name'], 'hermes_url': 'http://127.0.0.1:1234', 'hermes_api_key': 'private-hermes-key'} + async def ready(self, user): return True + async def remove(self, user): pass + +class Catalog: + unavailable = False + async def models(self): + if self.unavailable: raise RuntimeError('secret-upstream-key') + return [{'id': 'auto', 'name': 'auto', 'thinking_levels': ['none', 'high']}, + {'id': 'openai/test', 'name': 'openai/test', 'thinking_levels': ['none', 'high']}] + +@pytest.fixture +def env(tmp_path): + catalog = Catalog() + app = create_app(Settings(db_path=str(tmp_path/'accounts.db'), admin_password=PASSWORD, + service_secret=SECRET, allowed_origins=('https://testserver',), reconcile_interval=3600), Provisioner(), catalog) + with TestClient(app, base_url='https://testserver') as client: + yield client, app, catalog + +def login(client, name='admin'): + result = client.post('/api/auth/login', json={'username': name, 'password': PASSWORD}, headers={'Origin':'https://testserver'}) + assert result.status_code == 200 + return {'Authorization': 'Bearer ' + result.json()['token']}, result.json()['user']['id'] + +def test_authenticated_saved_settings_are_account_scoped(env): + client, app, catalog = env + assert client.get('/api/settings/model').status_code == 401 + headers, uid = login(client) + assert client.get('/api/settings/model', headers=headers).json()['settings'] == {'model':'auto', 'thinking_level':'none'} + saved = {'model':'openai/test', 'thinking_level':'high'} + assert client.put('/api/settings/model', headers=headers, json=saved).json()['settings'] == saved + created = client.post('/api/admin/users', headers=headers, json={'username':'other', 'display_name':'Other', 'password':PASSWORD}) + assert created.status_code == 201 + other, _ = login(client, 'other') + assert client.get('/api/settings/model', headers=other).json()['settings'] == {'model':'auto', 'thinking_level':'none'} + assert client.get('/api/settings/model', headers=headers).json()['settings'] == saved + app.state.accounts.update(uid, status='ready', hermes_url='http://127.0.0.1:1234', hermes_api_key='private-hermes-key') + response = client.get('/internal/users/'+uid, headers={'X-Control-Secret':SECRET}) + assert response.status_code == 200 + assert response.json()['model_settings'] == saved + assert 'secret' not in json.dumps(client.get('/api/settings/model', headers=headers).json()) + +@pytest.mark.parametrize('body', [ + {'model':'unlisted', 'thinking_level':'none'}, + {'model':'auto', 'thinking_level':'ultra'}, + {'model':'auto', 'thinking_level':'none', 'provider':'evil'}, + {'model':'auto', 'thinking_level':'none', 'user_id':'another'}, + {'model':'auto', 'thinking_level':{'enabled':False}}, +]) +def test_rejects_untrusted_model_effort_and_routing(env, body): + client, app, catalog = env + headers, uid = login(client) + assert client.put('/api/settings/model', headers=headers, json=body).status_code == 400 + assert app.state.accounts.get_model_settings(uid) == {'model':'auto', 'thinking_level':'none'} + +def test_csrf_and_catalog_failure_preserve_settings(env): + client, app, catalog = env + headers, uid = login(client) + saved = {'model':'openai/test', 'thinking_level':'high'} + assert client.put('/api/settings/model', headers=headers, json=saved).status_code == 200 + assert client.put('/api/settings/model', json=saved).status_code == 403 + assert client.put('/api/settings/model', headers={'Origin':'https://evil'}, json=saved).status_code == 403 + catalog.unavailable = True + get = client.get('/api/settings/model', headers=headers) + assert get.status_code == 200 and not get.json()['catalog_available'] + assert get.json()['settings'] == saved + put = client.put('/api/settings/model', headers=headers, json={'model':'auto','thinking_level':'none'}) + assert put.status_code == 503 and 'secret' not in put.text + assert app.state.accounts.get_model_settings(uid) == saved + +@pytest.mark.asyncio +async def test_catalog_projects_metadata_and_caches(): + calls = [] + def handle(request): + calls.append(request) + return httpx.Response(200,json={'data':[{'id':'auto','api_key':'NEVER'}, {'id':'openai/test'}, {'id':'bad model'}, {'id':'auto'}]}) + async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client: + catalog = ModelCatalog(base_url='https://trusted/v1', api_key='SECRET', client=client) + first = await catalog.models() + assert [m['id'] for m in first] == ['auto','openai/test'] + assert 'NEVER' not in json.dumps(first) + assert await catalog.models() == first and len(calls) == 1 + assert calls[0].headers['Authorization'] == 'Bearer SECRET' + assert str(calls[0].url) == 'https://trusted/v1/models' + +@pytest.mark.asyncio +async def test_catalog_errors_are_redacted(): + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda r:httpx.Response(401,text='SECRET'))) as client: + catalog = ModelCatalog(api_key='SECRET', client=client) + with pytest.raises(CatalogUnavailable) as exc: + await catalog.models() + assert 'SECRET' not in str(exc.value) diff --git a/tests/unit/test_model_settings_ui.py b/tests/unit/test_model_settings_ui.py new file mode 100644 index 0000000..50a0172 --- /dev/null +++ b/tests/unit/test_model_settings_ui.py @@ -0,0 +1,69 @@ +"""Offline Chromium checks of actual Hope assets with mocked account APIs.""" +import json +from pathlib import Path +from playwright.sync_api import sync_playwright, expect + +ROOT = Path(__file__).resolve().parents[2] + + +def test_model_settings_load_save_reload_and_failure(): + saved = {'model': 'auto', 'thinking_level': 'none'} + account = {'id':'alice','username':'alice','display_name':'Alice','role':'user','status':'ready','error':None} + calls = [] + available = True + models = [{'id':m,'name':m,'thinking_levels':['none','low','high']} for m in ['auto','openai/gpt-6.1-sol-subscription']] + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page(viewport={'width':390,'height':844}) + errors = [] + page.on('pageerror', lambda e: errors.append(str(e))) + def route(req): + nonlocal saved + url = req.request.url + if '/api/auth/me' in url: + req.fulfill(json={'user':account}) + elif '/api/settings/model' in url: + calls.append(req.request.method) + if req.request.method == 'PUT': + saved = req.request.post_data_json + req.fulfill(json={'settings':saved,'message':'Saved for your account. Applies to your next request.'}) + else: + req.fulfill(json={'settings':saved,'models':models if available else [],'catalog_available':available,'message':''}) + else: + name = url.split('/',3)[-1].split('?')[0] or 'index.html' + asset = ROOT/'web'/name + if asset.is_file(): + types = {'.html':'text/html','.js':'application/javascript','.css':'text/css','.svg':'image/svg+xml'} + req.fulfill(body=asset.read_bytes(),content_type=types.get(asset.suffix,'application/octet-stream')) + else: + req.fulfill(status=404,body='') + page.route('https://hope.test/**', route) + page.goto('https://hope.test/') + page.wait_for_load_state('networkidle') + assert calls == [] + page.locator('#settingsBtn').click() + expect(page.locator('#modelSelect')).to_be_enabled() + expect(page.locator('#saveModelSettings')).to_be_disabled() + page.locator('#modelSelect').select_option('openai/gpt-6.1-sol-subscription') + page.locator('#thinkingLevelSelect').select_option('high') + page.locator('#saveModelSettings').click() + expect(page.locator('#modelSettingsStatus')).to_contain_text('Saved') + assert saved == {'model':'openai/gpt-6.1-sol-subscription','thinking_level':'high'} + assert page.evaluate('document.documentElement.scrollWidth <= innerWidth') + page.locator('#closeSheetBtn').click() + page.reload(); page.wait_for_load_state('networkidle') + page.locator('#settingsBtn').click() + expect(page.locator('#modelSelect')).to_have_value(saved['model']) + expect(page.locator('#thinkingLevelSelect')).to_have_value('high') + page.screenshot(path='/tmp/hope-model-settings-mobile.png',full_page=True) + page.locator('#closeSheetBtn').click() + available = False + page.locator('#settingsBtn').click() + expect(page.locator('#modelSelect')).to_be_disabled() + expect(page.locator('#saveModelSettings')).to_be_disabled() + expect(page.locator('#retryModelSettings')).to_be_visible() + available = True + page.locator('#retryModelSettings').click() + expect(page.locator('#modelSelect')).to_be_enabled() + assert errors == [] + browser.close() diff --git a/web/app.js b/web/app.js index c778302..131c45f 100644 --- a/web/app.js +++ b/web/app.js @@ -586,6 +586,163 @@ function handleDataPacket(payload, participant, topic) { } } +// ── Account model settings (server saved; never browser persisted) ────────── +const modelSettingsForm = document.getElementById("modelSettingsForm"); +const modelSelect = document.getElementById("modelSelect"); +const thinkingLevelSelect = document.getElementById("thinkingLevelSelect"); +const saveModelSettingsBtn = document.getElementById("saveModelSettings"); +const retryModelSettingsBtn = document.getElementById("retryModelSettings"); +const modelSettingsStatus = document.getElementById("modelSettingsStatus"); +const thinkingLevelNote = document.getElementById("thinkingLevelNote"); +const THINKING_LEVELS = [ + ["none", "Off"], ["minimal", "Minimal"], ["low", "Low"], + ["medium", "Medium"], ["high", "High"], ["xhigh", "Extra high"], +]; +const THINKING_NOTE = "Thinking requests may be limited or ignored by the selected model."; +let settingsAccount = null; +let settingsGeneration = 0; +let settingsAbort = null; +let settingsBusy = false; +let modelCatalog = []; +let catalogAvailable = false; +let savedModelSettings = null; + +function accountIdentity() { + return HopeAuth.user ? String(HopeAuth.user.id) : null; +} +function settingsResponseIsCurrent(account, generation) { + return account !== null && account === accountIdentity() && account === settingsAccount && generation === settingsGeneration; +} +function settingsOption(value, label) { + const option = document.createElement("option"); + option.value = value; option.textContent = label; + return option; +} +function setSettingsStatus(message, state = "") { + modelSettingsStatus.textContent = message; + modelSettingsStatus.dataset.state = state; +} +function updateSettingsControls() { + const available = !!HopeAuth.user && catalogAvailable && !settingsBusy; + modelSelect.disabled = !available; + thinkingLevelSelect.disabled = !available; + const model = modelCatalog.find(item => item.id === modelSelect.value); + const valid = model && model.thinking_levels.includes(thinkingLevelSelect.value); + const changed = savedModelSettings && (modelSelect.value !== savedModelSettings.model || thinkingLevelSelect.value !== savedModelSettings.thinking_level); + saveModelSettingsBtn.disabled = !available || !valid || !changed; + retryModelSettingsBtn.disabled = settingsBusy; + saveModelSettingsBtn.textContent = settingsBusy && modelSettingsForm.dataset.operation === "save" ? "Saving…" : "Save"; + modelSettingsForm.setAttribute("aria-busy", String(settingsBusy)); +} +function renderThinkingLevels(preferred) { + const model = modelCatalog.find(item => item.id === modelSelect.value); + const levels = model?.thinking_levels || []; + thinkingLevelSelect.replaceChildren(); + for (const [value, label] of THINKING_LEVELS) { + if (levels.includes(value) || value === preferred) thinkingLevelSelect.append(settingsOption(value, label)); + } + if (!thinkingLevelSelect.options.length) thinkingLevelSelect.append(settingsOption("", "No thinking levels available")); + if ([...thinkingLevelSelect.options].some(option => option.value === preferred)) thinkingLevelSelect.value = preferred; + updateSettingsControls(); +} +function renderSavedModelSettings() { + modelSelect.replaceChildren(); + for (const model of modelCatalog) modelSelect.append(settingsOption(model.id, model.name || model.id)); + if (savedModelSettings && !modelCatalog.some(model => model.id === savedModelSettings.model)) { + modelSelect.prepend(settingsOption(savedModelSettings.model, `${savedModelSettings.model} (saved; unavailable)`)); + } + if (!modelSelect.options.length) modelSelect.append(settingsOption("", "Model catalog unavailable")); + if (savedModelSettings) modelSelect.value = savedModelSettings.model; + renderThinkingLevels(savedModelSettings?.thinking_level); +} +function resetModelSettings(user) { + settingsGeneration++; + settingsAbort?.abort(); settingsAbort = null; + settingsAccount = user ? String(user.id) : null; + settingsBusy = false; catalogAvailable = false; modelCatalog = []; savedModelSettings = null; + modelSettingsForm.dataset.operation = ""; + modelSelect.replaceChildren(settingsOption("", "Open settings to load models")); + thinkingLevelSelect.replaceChildren(settingsOption("", "Open settings to load thinking levels")); + thinkingLevelNote.textContent = THINKING_NOTE; + retryModelSettingsBtn.hidden = true; + setSettingsStatus("Open settings to load your saved preferences."); + updateSettingsControls(); +} +function readSavedSettings(data) { + if (!data?.settings || typeof data.settings.model !== "string" || !THINKING_LEVELS.some(([level]) => level === data.settings.thinking_level)) { + throw new Error("The server did not return valid saved settings."); + } + return { model: data.settings.model, thinking_level: data.settings.thinking_level }; +} +async function loadModelSettings() { + const account = accountIdentity(); + if (!account || settingsBusy) return; + if (settingsAccount !== account) resetModelSettings(HopeAuth.user); + const generation = ++settingsGeneration; + settingsAbort?.abort(); settingsAbort = new AbortController(); + settingsBusy = true; modelSettingsForm.dataset.operation = "load"; + retryModelSettingsBtn.hidden = true; + setSettingsStatus("Loading your saved model settings…"); updateSettingsControls(); + try { + const data = await HopeAuth.request("/api/settings/model", { signal: settingsAbort.signal }); + if (!settingsResponseIsCurrent(account, generation)) return; + savedModelSettings = readSavedSettings(data); + modelCatalog = Array.isArray(data.models) ? data.models.filter(model => typeof model?.id === "string" && Array.isArray(model.thinking_levels)).map(model => ({ + id: model.id, name: typeof model.name === "string" ? model.name : model.id, + thinking_levels: model.thinking_levels.filter(level => THINKING_LEVELS.some(([value]) => value === level)), + })) : []; + catalogAvailable = data.catalog_available === true && modelCatalog.length > 0; + thinkingLevelNote.textContent = typeof data.thinking_note === "string" && data.thinking_note ? data.thinking_note : THINKING_NOTE; + renderSavedModelSettings(); + retryModelSettingsBtn.hidden = catalogAvailable; + setSettingsStatus(catalogAvailable ? (data.message || "Your saved settings are loaded. Choose preferences, then Save.") : `${data.message || "Model catalog unavailable."} Your saved preferences are shown; retry to make changes.`, catalogAvailable ? "" : "error"); + } catch (error) { + if (!settingsResponseIsCurrent(account, generation)) return; + catalogAvailable = false; + retryModelSettingsBtn.hidden = false; + // Keep this account's last saved display when a refresh fails. + renderSavedModelSettings(); + setSettingsStatus(`Could not load model settings: ${error.message} Retry to load your saved preferences.`, "error"); + } finally { + if (settingsResponseIsCurrent(account, generation)) { + settingsBusy = false; settingsAbort = null; updateSettingsControls(); + } + } +} +modelSelect.addEventListener("change", () => { + const previous = thinkingLevelSelect.value; + const model = modelCatalog.find(item => item.id === modelSelect.value); + renderThinkingLevels(model?.thinking_levels.includes(previous) ? previous : model?.thinking_levels[0]); + setSettingsStatus("Unsaved changes. Save to apply them to your account."); +}); +thinkingLevelSelect.addEventListener("change", () => { + updateSettingsControls(); setSettingsStatus("Unsaved changes. Save to apply them to your account."); +}); +retryModelSettingsBtn.addEventListener("click", loadModelSettings); +modelSettingsForm.addEventListener("submit", async event => { + event.preventDefault(); updateSettingsControls(); + if (saveModelSettingsBtn.disabled) return; + const account = accountIdentity(), generation = ++settingsGeneration; + const settings = { model: modelSelect.value, thinking_level: thinkingLevelSelect.value }; + settingsAbort = new AbortController(); settingsBusy = true; modelSettingsForm.dataset.operation = "save"; + retryModelSettingsBtn.hidden = true; + setSettingsStatus("Saving your model settings…"); updateSettingsControls(); + try { + const data = await HopeAuth.request("/api/settings/model", { method: "PUT", body: JSON.stringify(settings), signal: settingsAbort.signal }); + if (!settingsResponseIsCurrent(account, generation)) return; + savedModelSettings = readSavedSettings(data); renderSavedModelSettings(); + setSettingsStatus(data.message || "Saved. Your next request will use these account settings; your conversation continues uninterrupted.", "success"); + } catch (error) { + if (!settingsResponseIsCurrent(account, generation)) return; + retryModelSettingsBtn.hidden = false; + setSettingsStatus(`Could not save settings: ${error.message} Your changes are not confirmed saved. Try Save again, or Retry to reload saved preferences.`, "error"); + } finally { + if (settingsResponseIsCurrent(account, generation)) { + settingsBusy = false; settingsAbort = null; updateSettingsControls(); + } + } +}); + // ── Voice settings sheet ──────────────────────────────────────────────────── function currentVoiceId() { const saved = localStorage.getItem("preferred_voice"); @@ -631,13 +788,16 @@ function selectVoice(voiceId) { } function openSheet() { + if (!HopeAuth.user) return; renderVoiceList(); settingsSheet.hidden = false; settingsSheet.inert = false; document.getElementById("authenticatedApp").inert = true; sheetBackdrop.hidden = false; settingsSheet.classList.add("open"); settingsBtn.setAttribute("aria-expanded", "true"); - settingsSheet.querySelector(".voice-card")?.focus(); + loadModelSettings(); + // The catalog is loading, so the first stable enabled control is Close. + sheetFocusableControls()[0]?.focus(); } function closeSheet() { @@ -649,15 +809,22 @@ function closeSheet() { settingsBtn.focus(); } +function sheetFocusableControls() { + return [...settingsSheet.querySelectorAll('button, select, input, textarea, a[href], [tabindex]')].filter(el => + !el.disabled && el.tabIndex >= 0 && !el.closest('[hidden], [inert]') && + el.getClientRects().length > 0 && getComputedStyle(el).visibility !== "hidden" + ); +} settingsBtn.setAttribute("aria-expanded", "false"); settingsBtn.addEventListener("click", openSheet); closeSheetBtn.addEventListener("click", closeSheet); sheetBackdrop.addEventListener("click", closeSheet); document.addEventListener("keydown", (event) => { if (event.key === "Tab" && settingsSheet.classList.contains("open")) { - const focusable = [...settingsSheet.querySelectorAll('button:not(:disabled), [tabindex="0"]')]; + const focusable = sheetFocusableControls(); const first = focusable[0], last = focusable[focusable.length - 1]; - if (event.shiftKey && document.activeElement === first) { event.preventDefault(); last?.focus(); } + if (!focusable.includes(document.activeElement)) { event.preventDefault(); first?.focus(); } + else if (event.shiftKey && document.activeElement === first) { event.preventDefault(); last?.focus(); } else if (!event.shiftKey && document.activeElement === last) { event.preventDefault(); first?.focus(); } } if (event.key !== "Escape") return; @@ -961,6 +1128,7 @@ function sendModeMessage(mode, targetRoom) { HopeAuth.onChange((user, changed) => { if (changed || !user || user.status !== "ready") stopConversation(); + if (changed || settingsAccount !== (user ? String(user.id) : null)) resetModelSettings(user); if (changed) { historyKey = user ? `hope_history:${location.origin}:${encodeURIComponent(user.id)}` : null; history = loadHistory(); renderHistory(); progress.clear(); tasksList.replaceChildren(); tasksBadge.hidden = true; diff --git a/web/index.html b/web/index.html index df59dac..29fa60b 100644 --- a/web/index.html +++ b/web/index.html @@ -103,7 +103,7 @@ - @@ -146,14 +146,34 @@

Hope’s settings

-

Your voice preference applies to this conversation. Reasoning is managed by your private assistant.

+

Your voice preference applies to this conversation.

-

Voice

-
+
+
+

Model and thinking

+

Saved for your account. Applies to your next request, including in an active conversation, without interrupting it.

+
+ + +
+
+ + +

Thinking requests may be limited or ignored by the selected model.

+
+

Open settings to load your saved preferences.

+
+ + +
+
+

Voice

+
+