feat: qualify native Hermes research and private conversation exports

control/chat_exports.py

@@ -0,0 +1,53 @@
+"""Private bounded conversation exports; no public sharing or provider execution."""
+import asyncio
+import json
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
+
+from .app import current_user
+
+MAX_EXPORT_BYTES = 16 * 1024 * 1024
+MAX_EXPORT_MESSAGES = 10_000
+
+
+def export_conversation(store, owner, identifier, format='json', byte_limit=MAX_EXPORT_BYTES):
+    if format not in ('json', 'markdown'):
+        raise HTTPException(422, 'Export format must be json or markdown')
+    with store.lock:
+        try:
+            conversation = store._conversation(store._row('chat_conversations', owner, identifier))
+        except (KeyError, ValueError):
+            raise HTTPException(404, 'Conversation not found') from None
+        count, content_bytes = store.db.execute('SELECT COUNT(*),COALESCE(SUM(length(CAST(content AS BLOB))),0) FROM chat_messages WHERE owner=? AND conversation_id=?', (owner, identifier)).fetchone()
+        if count > MAX_EXPORT_MESSAGES or content_bytes > byte_limit:
+            raise HTTPException(413, 'Conversation export limit exceeded')
+        messages = [store._message(row) for row in store.db.execute('SELECT * FROM chat_messages WHERE owner=? AND conversation_id=? ORDER BY position', (owner, identifier))]
+    if format == 'json':
+        # Only public projection; no Hermes IDs, API URLs, credentials or tool events.
+        encoded = json.dumps({'version': 1, 'conversation': conversation, 'messages': messages}, ensure_ascii=False, indent=2).encode('utf-8')
+        mime, suffix = 'application/json', '.json'
+    else:
+        # Preserve original message text/citations. Always attachment+plain text:
+        # user markdown/HTML is never executed or rendered by this API.
+        sections = ['# ' + conversation['title'], '']
+        for message in messages:
+            sections += ['## ' + message['role'].capitalize() + ' — ' + message['status'], '', message['content'], '']
+            if message['attachments']:
+                sections += ['Attachments (IDs; private bytes not embedded): ' + ', '.join(message['attachments']), '']
+        encoded = '\n'.join(sections).encode('utf-8')
+        mime, suffix = 'text/plain', '.md'
+    if len(encoded) > byte_limit:
+        raise HTTPException(413, 'Conversation export limit exceeded')
+    return encoded, mime, 'conversation-' + conversation['id'] + suffix
+
+
+def create_export_router():
+    router = APIRouter()
+
+    @router.get('/api/conversations/{identifier}/export')
+    async def conversation_export(identifier: str, request: Request, format: str = 'json', user=Depends(current_user)):
+        data, mime, filename = await asyncio.to_thread(export_conversation, request.app.state.chat_store, user['id'], identifier, format)
+        await current_user(request)  # Revocation during export denies final response.
+        return Response(data, media_type=mime, headers={'Content-Disposition': 'attachment; filename="' + filename + '"',
+                                                       'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff'})
+    return router

control/chat_features.py

@@ -361,6 +361,36 @@ class FeatureProvider:
             raise HTTPException(503, 'Hermes not ready')
         return user
 
+    async def toolset_status(self, owner):
+        user = await self._account(owner)
+        try:
+            response = await self.http.get(user['hermes_url'].rstrip('/') + '/v1/toolsets', headers={'Authorization': 'Bearer ' + user['hermes_api_key']})
+            if response.status_code != 200:
+                raise ValueError()
+            body = response.json()
+            if not isinstance(body, dict) or not isinstance(body.get('data'), list):
+                raise ValueError()
+            return [{'name': row['name'], 'enabled': row.get('enabled') is True, 'configured': row.get('configured') is True,
+                     'tools': [tool for tool in row.get('tools', []) if isinstance(tool, str)][:200]}
+                    for row in body['data'][:100] if isinstance(row, dict) and isinstance(row.get('name'), str) and isinstance(row.get('tools'), list)]
+        except Exception:
+            raise HTTPException(503, 'Hermes tool capability status unavailable') from None
+
+    async def research_status(self, owner):
+        toolsets = await self.toolset_status(owner)
+        available = any(row['name'] == 'web' and row['enabled'] and row['configured'] and
+                        {'web_search', 'web_extract'} <= set(row['tools']) for row in toolsets)
+        return {'available': available, 'mode': 'hermes_tools' if available else None,
+                'reason': None if available else 'Hermes web search and extraction must be enabled and configured'}
+
+    async def research_instruction(self, owner):
+        status = await self.research_status(owner)
+        if not status['available']:
+            raise HTTPException(409, 'Research tools are unavailable for this account')
+        return ('Research mode requested by the user. Use your Hermes web search and extraction tools to investigate the question. '
+                'Prefer primary sources, corroborate material claims, distinguish evidence from uncertainty, and cite source URLs in the final answer. '
+                'Do not claim sources were checked if tools failed or access was unavailable. Complete this research in this same Hermes run.')
+
     async def feature_capabilities(self, owner):
         await self._account(owner)
         settings = await asyncio.to_thread(self.accounts.get_model_settings, owner)
@@ -372,8 +402,12 @@ class FeatureProvider:
                 return True
             except HTTPException:
                 return False
+        try:
+            research = (await self.research_status(owner))['available']
+        except HTTPException:
+            research = False
         return {'scheduled_tasks': True, 'memory_controls': await asyncio.to_thread(memory_available),
-                'image_input': settings['model'] == 'auto', 'connectors': False, 'research': False}
+                'image_input': settings['model'] == 'auto', 'connectors': False, 'research': research}
 
     async def feature_request(self, owner, method, path, *, json=None, params=None):
         user = await self._account(owner)
@@ -712,12 +746,24 @@ def create_feature_router(store, provider=None):
     @router.get('/api/connectors')
     async def connectors(user=Depends(current_user)):
         # The pinned gateway has no connector management API. Do not invent one.
-        return {'connectors': [], 'available': False, 'reason': 'Hermes connector management API is not qualified'}
+        statuses = []
+        if provider is not None and hasattr(provider, 'toolset_status'):
+            try:
+                statuses = await provider.toolset_status(user['id'])
+            except HTTPException:
+                pass
+        return {'connectors': [], 'available': False, 'reason': 'Pinned Hermes has no connector credential/settings write API',
+                'toolsets': [{'name': row['name'], 'enabled': row['enabled'], 'configured': row['configured']} for row in statuses]}
 
     @router.get('/api/research')
     async def research(user=Depends(current_user)):
         # Research remains normal Hermes execution, never a Hope reasoning loop.
-        return {'available': False, 'reason': 'Dedicated research mode is not qualified; normal Hermes tool use remains available'}
+        if provider is not None and hasattr(provider, 'research_status'):
+            try:
+                return await provider.research_status(user['id'])
+            except HTTPException:
+                pass
+        return {'available': False, 'mode': None, 'reason': 'Hermes research tool availability could not be qualified'}
 
     return router
 

docs/CHAT_FEATURES_CONTRACT.md

@@ -49,8 +49,17 @@ Revision SHA256 hex content digest. Empty content explicitly clears that section
 
 ## Honest statuses and sync
 
-- GET `/api/connectors` → `{connectors:[],available:false,reason}`. No qualified connector management API; do not present fabricated working integration switches.
+- GET `/api/connectors` → `{connectors:[],available:false,reason,toolsets:[{name,enabled,configured}]}`. Read-only native tool configuration statuses from exact authenticated `/v1/toolsets`. No qualified connector credential/settings write API; do not present fabricated working OAuth/integration switches. Raw credentials/configuration are never returned.
-- GET `/api/research` → `{available:false,reason}`. Normal Hermes web/browser tools remain available; dedicated research-mode API is not qualified.
+- GET `/api/research` → `{available:bool,mode:"hermes_tools"|null,reason:string|null}`. Enabled only when the actual account manifest advertises enabled/configured `web` with both `web_search` and `web_extract`. Capability lookup failure is unavailable, never optimistic true. Native research means one ordinary Hermes run with source-verification/citation instruction, not a second Hope reasoning loop or special upstream research endpoint.
+- Core message extension target POST `/api/conversations/{id}/messages` adds `mode?:"normal"|"research"`. Core must persist mode in idempotency payload and run settings; research calls `FeatureProvider.research_instruction(owner)` and supplies the returned static evidence/citations instruction to that same native Hermes run. Unqualified tools409. This module does not itself edit core message acceptance.
 - GET `/api/features/sync?after=0` → `{changes:[{sequence,kind:"attachment"|"artifact",id,deleted,created_at}],cursor}`; max200 per response, owner-filtered tombstones. This cursor is independent of core conversation sync.
 
-Core merges `local_capabilities(store)` and account-qualified `FeatureProvider.feature_capabilities(owner)` into its one `/api/capabilities`. Local helper reports image/tasks/memory false by default; provider qualifies jobs, managed auto vision, and available fixed memory storage. Constructor `FeatureStore(accounts, root_path, limits=None)`; router `create_feature_router(store, provider=None)`; default `FeatureProvider(accounts,http_client=None,state_dir=None)` has `aclose()` which app shutdown must call. Memory root defaults HOPE_HERMES_DATA_DIR matching provisioner. No live/deploy qualification performed here.
+Core merges `local_capabilities(store)` and account-qualified `FeatureProvider.feature_capabilities(owner)` into its one `/api/capabilities`. Local helper reports image/tasks/memory false by default; provider qualifies jobs, managed auto vision, and available fixed memory storage. Constructor `FeatureStore(accounts, root_path, limits=None)`; router `create_feature_router(store, provider=None)`; default `FeatureProvider(accounts,http_client=None,state_dir=None)` has `aclose()` which app shutdown must call. Memory root defaults HOPE_HERMES_DATA_DIR matching provisioner. Research source verified pinned `749220ef` gateway handler `_handle_toolsets` and exact web toolset expansion. No live/deploy qualification performed here.
+
+## Private conversation exports
+
+`create_export_router()` from `control.chat_exports` uses injected `app.state.chat_store`.
+
+- GET `/api/conversations/{id}/export?format=json|markdown` → authenticated attachment download. Defaults json. JSON `{version:1,conversation,messages:[]}` includes entire chronological message history, not a paginated first page. Only existing public projections, no private Hermes IDs, tool reasoning/events, account credentials or URLs. Markdown preserves original text/citations, exports as text/plain `.md` attachment rather than executable HTML.
+- Full export is bounded16MiB encoded output and10000 messages; oversized413, unsupported format422, cross-account/deleted404, revoked401. Session revalidated before final response. Attachment IDs preserved but private upload bytes not embedded. No public share links, remote upload or public artifacts.
+- Existing artifact export remains `/api/artifacts/{id}/export?revision=N`; exports selected exact stored content. Native model-generated artifacts require core capture/edit linkage, not an invented Hermes file API.

tests/unit/test_chat_exports.py

@@ -0,0 +1,64 @@
+"""Offline owned exports; no Hermes/LiveKit/deployment calls."""
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+
+from control.accounts import Accounts
+from control.app import Settings
+from control.chat_exports import create_export_router, export_conversation
+from control.chat_store import ChatStore
+
+
+@pytest.fixture
+def env(tmp_path):
+    accounts = Accounts(str(tmp_path / 'accounts.db'))
+    users = [accounts.create(name, name, 'user', 'correct-horse-password') for name in ('alice', 'bob')]
+    tokens = [accounts.issue_session(u['id'], 3600, u['password_hash'])[0] for u in users]
+    store = ChatStore(accounts)
+    conversation = store.create_conversation(users[0]['id'], title='Private')
+    store.add_message(users[0]['id'], conversation['id'], 'user', 'Original question')
+    store.add_message(users[0]['id'], conversation['id'], 'assistant', '<script>untrusted</script> [Source](https://example.test)')
+    app = FastAPI()
+    app.state.accounts = accounts
+    app.state.settings = Settings()
+    app.state.chat_store = store
+    app.include_router(create_export_router())
+    with TestClient(app) as client:
+        yield accounts, store, client, users, tokens, conversation
+    accounts.close()
+
+
+def test_full_projection_and_safe_markdown(env):
+    accounts, store, client, users, tokens, conversation = env
+    headers = {'Authorization': 'Bearer ' + tokens[0]}
+    url = '/api/conversations/' + conversation['id'] + '/export'
+    response = client.get(url, headers=headers)
+    assert response.status_code == 200
+    document = response.json()
+    assert len(document['messages']) == 2
+    assert document['messages'][0]['content'] == 'Original question'
+    assert 'hermes_session_id' not in response.text
+    assert response.headers['cache-control'] == 'no-store'
+    assert response.headers['content-disposition'].startswith('attachment;')
+    markdown = client.get(url + '?format=markdown', headers=headers)
+    assert markdown.status_code == 200
+    assert markdown.headers['content-type'].startswith('text/plain')
+    assert '[Source](https://example.test)' in markdown.text
+    assert '<script>untrusted</script>' in markdown.text
+    assert client.get(url + '?format=html', headers=headers).status_code == 422
+
+
+def test_owned_revoked_deleted_and_bounded(env):
+    accounts, store, client, users, tokens, conversation = env
+    url = '/api/conversations/' + conversation['id'] + '/export'
+    assert client.get(url).status_code == 401
+    assert client.get(url, headers={'Authorization': 'Bearer ' + tokens[1]}).status_code == 404
+    with pytest.raises(HTTPException) as error:
+        export_conversation(store, users[0]['id'], conversation['id'], byte_limit=4)
+    assert error.value.status_code == 413
+    accounts.revoke(tokens[0])
+    assert client.get(url, headers={'Authorization': 'Bearer ' + tokens[0]}).status_code == 401
+    store.delete_conversation(users[0]['id'], conversation['id'])
+    with pytest.raises(HTTPException) as error:
+        export_conversation(store, users[0]['id'], conversation['id'])
+    assert error.value.status_code == 404

tests/unit/test_chat_features.py

@@ -310,3 +310,31 @@ def test_pdf_subprocess_limits_and_admission(env, monkeypatch):
     finally:
         env.store.upload_admission.release()
         env.store.upload_admission.release()
+
+
+def test_research_exact_tool_manifest_and_status_projection(env):
+    import httpx
+    manifest = {'object': 'list', 'platform': 'api_server', 'data': [
+        {'name': 'web', 'enabled': True, 'configured': True, 'tools': ['web_search', 'web_extract'], 'secret': 'never expose'}]}
+    requests = []
+    def handle(request):
+        requests.append(request)
+        assert request.url.path == '/v1/toolsets'
+        return httpx.Response(200, json=manifest)
+    http = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+    env.accounts.update(env.users[0]['id'], hermes_url='http://127.0.0.1:20000', hermes_api_key='private')
+    provider = FeatureProvider(env.accounts, http_client=http)
+    assert asyncio.run(provider.research_status(env.users[0]['id'])) == {'available': True, 'mode': 'hermes_tools', 'reason': None}
+    instruction = asyncio.run(provider.research_instruction(env.users[0]['id']))
+    assert 'same Hermes run' in instruction and 'cite source URLs' in instruction
+    assert requests[-1].headers['authorization'] == 'Bearer private'
+    assert 'secret' not in str(asyncio.run(provider.toolset_status(env.users[0]['id'])))
+    manifest['data'][0]['configured'] = False
+    assert not asyncio.run(provider.research_status(env.users[0]['id']))['available']
+    with pytest.raises(HTTPException) as error:
+        asyncio.run(provider.research_instruction(env.users[0]['id']))
+    assert error.value.status_code == 409
+    manifest['data'][0]['configured'] = True
+    manifest['data'][0]['tools'] = ['web_search']
+    assert not asyncio.run(provider.research_status(env.users[0]['id']))['available']
+    asyncio.run(http.aclose())