feat: add Hermes WebUI provisioning and English Dragon HD Omni voices

control/provisioner.py

@@ -1,5 +1,6 @@
 """Trusted Docker-UDS control plane for independent Hermes accounts.
 
+Hermes uses host networking for LAN egress with per-account loopback listeners.
 Only this service may receive the host Docker socket. Bind-source paths must be
 identical in the control service and Docker host (mount the runtime root there).
 The official image's root/s6 boot is deliberately bypassed: a nonroot Python
@@ -14,8 +15,11 @@ import os
 from pathlib import Path
 import re
 import secrets
+import socket
 import tempfile
 
+import yaml
+
 import httpx
 
 
@@ -30,7 +34,7 @@ class OwnershipError(ProvisioningError):
 # Run as the image's hermes UID without executing root-only s6 startup. No source
 # checkout/profile is copied: only the image's public bundled skills are seeded.
 _BOOTSTRAP = """
-import os, pathlib, shutil
+import os, pathlib, shutil, subprocess, signal, time, ssl
 home = pathlib.Path('/opt/data')
 for name in ('home', 'sessions', 'memories', 'skills', 'cron', 'hooks', 'logs', 'cache'):
     (home / name).mkdir(mode=0o700, parents=True, exist_ok=True)
@@ -39,7 +43,55 @@ if source.is_dir() and not (home / '.hope-skills-seeded').exists():
     shutil.copytree(source, home / 'skills', dirs_exist_ok=True)
     (home / '.hope-skills-seeded').touch(mode=0o600)
 os.makedirs('/tmp/hermes-runtime', mode=0o700, exist_ok=True)
-os.execv('/opt/hermes/.venv/bin/hermes', ['hermes', 'gateway', 'run'])
+if os.environ.get('HOPE_WEBUI_ENABLED') != 'true':
+    os.execv('/opt/hermes/.venv/bin/hermes', ['hermes', 'gateway', 'run'])
+# Generate a private per-account certificate once; never start without usable TLS.
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+import datetime, ipaddress
+tls = home / 'webui-tls'
+tls.mkdir(mode=0o700, exist_ok=True)
+cert, key = tls / 'cert.pem', tls / 'key.pem'
+if not cert.exists() and not key.exists():
+    private = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'Hope Hermes WebUI')])
+    now = datetime.datetime.now(datetime.timezone.utc)
+    names = [x509.DNSName('localhost'), x509.IPAddress(ipaddress.ip_address('127.0.0.1')),
+             x509.IPAddress(ipaddress.ip_address(os.environ['HERMES_WEBUI_HOST']))]
+    certificate = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject)
+        .public_key(private.public_key()).serial_number(x509.random_serial_number())
+        .not_valid_before(now - datetime.timedelta(minutes=5))
+        .not_valid_after(now + datetime.timedelta(days=3650))
+        .add_extension(x509.SubjectAlternativeName(names), critical=False).sign(private, hashes.SHA256()))
+    key.write_bytes(private.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
+                                         serialization.NoEncryption()))
+    os.chmod(key, 0o600)
+    cert.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
+ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER).load_cert_chain(str(cert), str(key))
+children = []
+def stop(signum=None, frame=None):
+    for child in children:
+        if child.poll() is None:
+            child.terminate()
+signal.signal(signal.SIGTERM, stop)
+signal.signal(signal.SIGINT, stop)
+try:
+    children.append(subprocess.Popen(['/opt/hermes/.venv/bin/hermes', 'gateway', 'run']))
+    children.append(subprocess.Popen(['/opt/hermes/.venv/bin/python', '/opt/hermes-webui/bootstrap.py',
+        '--foreground', '--no-browser', '--skip-agent-install']))
+    while all(child.poll() is None for child in children):
+        time.sleep(.5)
+finally:
+    stop()
+    for child in children:
+        try:
+            child.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            child.kill()
+            child.wait()
+raise SystemExit(1)
 """
 
 
@@ -56,8 +108,19 @@ class Provisioner:
     def __init__(self, state_dir=None, *, docker_socket=None, image=None,
                  upstream_api_key=None, docker_client=None, health_client=None,
                  ready_timeout=60, poll_interval=.5, runtime_uid=10000,
-                 runtime_gid=10000, memory_bytes=2 * 1024 ** 3, nano_cpus=2_000_000_000):
+                 runtime_gid=10000, api_port_range=None, memory_bytes=2 * 1024 ** 3, nano_cpus=2_000_000_000):
+        bounds = api_port_range or os.environ.get('HERMES_API_PORT_RANGE', '20000-29999')
+        try:
+            if isinstance(bounds, str):
+                bounds = tuple(map(int, bounds.split('-')))
+            if len(bounds) != 2 or any(type(port) is not int for port in bounds) or not 1024 <= bounds[0] <= bounds[1] <= 65535:
+                raise ValueError()
+        except (TypeError, ValueError) as exc:
+            raise ValueError('Invalid Hermes API port range') from exc
+        self.api_port_range = bounds
         self.state_dir = Path(state_dir or os.environ.get('HOPE_HERMES_DATA_DIR', '.runtime/hermes')).absolute()
+        self.webui_enabled = os.environ.get('HOPE_WEBUI_ENABLED', 'false').lower() == 'true'
+        self.webui_host = os.environ.get('HERMES_WEBUI_HOST', '192.168.86.2')
         self.image = image or os.environ.get('HERMES_IMAGE', 'nousresearch/hermes-agent@sha256:d4da4a40cd7a28aba983775d9fd31d94cbf153eeb0cb9e844d6d0f612b7c24db')
         self.upstream_api_key = upstream_api_key if upstream_api_key is not None else os.environ.get('HERMES_UPSTREAM_API_KEY', '')
         self.ready_timeout, self.poll_interval = ready_timeout, poll_interval
@@ -72,6 +135,48 @@ class Provisioner:
             base_url='http://docker', timeout=20, trust_env=False)
         self.health = health_client or httpx.AsyncClient(timeout=3, trust_env=False, follow_redirects=False)
         self._locks = {}
+        self._ports_lock = asyncio.Lock()
+
+    def _ports(self, webui=False):
+        path = self.state_dir / ('webui-ports.json' if webui else 'api-ports.json')
+        if path.is_symlink():
+            raise ProvisioningError('Port registry cannot be a symlink')
+        try:
+            ports = json.loads(path.read_text())
+        except FileNotFoundError:
+            return {}
+        if not isinstance(ports, dict) or any(
+                not re.fullmatch(r'[0-9a-f]{32}', uid) or type(port) is not int
+                or not 1024 <= port <= 65535 for uid, port in ports.items()):
+            raise ProvisioningError('Invalid Hermes port registry')
+        if len(set(ports.values())) != len(ports):
+            raise ProvisioningError('Duplicate Hermes API ports')
+        return ports
+
+    async def _port(self, user, webui=False):
+        # One control process owns this registry, including concurrent accounts.
+        async with self._ports_lock:
+            ports = self._ports(webui)
+            if user['id'] in ports:
+                return ports[user['id']]
+            used = set(ports.values()) | set(self._ports(not webui).values())
+            bounds = tuple(map(int, os.environ.get('HERMES_WEBUI_PORT_RANGE', '30000-30999').split('-'))) if webui else self.api_port_range
+            if len(bounds) != 2 or not 1024 <= bounds[0] <= bounds[1] <= 65535:
+                raise ProvisioningError('Invalid WebUI port range')
+            for port in range(bounds[0], bounds[1] + 1):
+                if port in used:
+                    continue
+                # Control must run in the Docker host network namespace.
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+                    try:
+                        probe.bind((self.webui_host if webui else '127.0.0.1', port))
+                    except OSError:
+                        continue
+                ports[user['id']] = port
+                self._directory(self.state_dir)
+                self._write_private(self.state_dir / ('webui-ports.json' if webui else 'api-ports.json'), json.dumps(ports) + '\n')
+                return port
+            raise ProvisioningError('No available Hermes API port')
 
     async def aclose(self):
         if self._own_docker:
@@ -147,7 +252,7 @@ class Provisioner:
             raise ProvisioningError('Invalid persisted Hermes API credential')
         return key
 
-    def _prepare(self, user, exists):
+    def _prepare(self, user, exists, port):
         private, home, workspace = self._paths(user)
         self._directory(self.state_dir)
         for path in (private, home, workspace):
@@ -161,9 +266,23 @@ class Provisioner:
         config = home / 'config.yaml'
         if not config.exists():
             # JSON is a YAML subset; no credentials are placed in configuration.
-            self._write_private(config, json.dumps(self.config(), indent=2) + '\n')
+            self._write_private(config, json.dumps(self.config(port), indent=2) + '\n')
         elif config.is_symlink():
             raise ProvisioningError('Hermes config cannot be a symlink')
+        else:
+            # Preserve user configuration; only control-owned listener settings change.
+            try:
+                value = yaml.safe_load(config.read_text())
+                if not isinstance(value, dict):
+                    raise ValueError()
+                server = value.setdefault('gateway', {}).setdefault('api_server', {})
+                if not isinstance(server, dict):
+                    raise ValueError()
+                if server.get('host') != '127.0.0.1' or server.get('port') != port:
+                    server.update(host='127.0.0.1', port=port)
+                    self._write_private(config, yaml.safe_dump(value, sort_keys=False))
+            except (ValueError, TypeError, AttributeError, yaml.YAMLError) as exc:
+                raise ProvisioningError('Invalid Hermes runtime configuration') from exc
         if os.geteuid() == 0:
             # Only seeded directory roots/config, never recursive retained data.
             for path in (home, workspace, config):
@@ -173,7 +292,7 @@ class Provisioner:
         return key, home, workspace
 
     @staticmethod
-    def config():
+    def config(port=20000):
         model = os.environ.get('HERMES_UPSTREAM_MODEL', 'auto')
         return {
             'model': {'default': model, 'provider': 'custom:hope-upstream'},
@@ -184,38 +303,54 @@ class Provisioner:
             'terminal': {'backend': 'local', 'cwd': '/workspace'},
             'platform_toolsets': {'api_server': ['terminal', 'file', 'web', 'browser', 'memory', 'skills', 'delegation']},
             'gateway': {'multiplex_profiles': False,
-                        'api_server': {'enabled': True, 'host': '0.0.0.0', 'port': 8642,
+                        'api_server': {'enabled': True, 'host': '127.0.0.1', 'port': port,
                                        'max_concurrent_runs': 1, 'tool_progress_events': True}},
         }
 
-    async def _network(self, user):
+    async def _remove_legacy_network(self, user):
         name = 'hope-hermes-net-' + user['id']
         response = await self._request('GET', f'/networks/{name}', allowed=(200, 404))
-        if response.status_code == 404:
-            await self._request('POST', '/networks/create', allowed=(201, 409), json={
-                'Name': name, 'Driver': 'bridge', 'CheckDuplicate': True,
-                'Labels': {'hope.managed': 'true', 'hope.user_id': user['id']}})
-            response = await self._request('GET', f'/networks/{name}')
-        network = response.json()
-        self._owned(network, user)
-        if network.get('Driver') != 'bridge' or network.get('Internal'):
-            raise ProvisioningError('Owned Hermes network is not an outbound-capable dedicated bridge')
-        return name
+        if response.status_code == 200:
+            network = response.json()
+            self._owned(network, user)
+            await self._request('DELETE', f'/networks/{network["Id"]}', allowed=(204, 404))
 
-    def _spec(self, user, key, home, workspace, network):
+    def _spec(self, user, key, home, workspace, port):
+        extra_env = []
+        if self.webui_enabled:
+            private, _, _ = self._paths(user)
+            password_path = private / 'webui-password'
+            if password_path.is_symlink():
+                raise ProvisioningError('WebUI password cannot be a symlink')
+            if not password_path.exists():
+                self._write_private(password_path, secrets.token_urlsafe(32) + '\n')
+            password = password_path.read_text().strip()
+            if not password or '\n' in password or '\r' in password:
+                raise ProvisioningError('WebUI password must be nonempty and single-line')
+            extra_env = ['HOPE_WEBUI_ENABLED=true', f'HERMES_WEBUI_HOST={self.webui_host}',
+                f'HERMES_WEBUI_PORT={self._ports(True)[user["id"]]}',
+                f'HERMES_WEBUI_PASSWORD={password}', f'HERMES_WEBUI_COOKIE_NAME=hope_webui_{user["id"]}',
+                'HERMES_WEBUI_SECURE=1', 'HERMES_WEBUI_SKIP_ONBOARDING=1',
+                'HERMES_WEBUI_PRESERVE_ENV=1', 'HERMES_WEBUI_AUTO_INSTALL=0',
+                'HERMES_WEBUI_SESSION_TTL=86400', 'HERMES_CONFIG_PATH=/opt/data/config.yaml',
+                'HERMES_WEBUI_DEFAULT_WORKSPACE=/workspace', 'HERMES_WEBUI_SERVER_CWD=/workspace',
+                'HERMES_WEBUI_STATE_DIR=/opt/data/webui',
+                'HERMES_WEBUI_TLS_CERT=/opt/data/webui-tls/cert.pem',
+                'HERMES_WEBUI_TLS_KEY=/opt/data/webui-tls/key.pem',
+                'HERMES_WEBUI_CHAT_BACKEND=gateway', f'HERMES_WEBUI_GATEWAY_BASE_URL=http://127.0.0.1:{port}',
+                f'HERMES_WEBUI_GATEWAY_API_KEY={key}', 'HERMES_WEBUI_GATEWAY_USE_RUNS_API=true']
         return {
             'Image': self.image, 'User': f'{self.runtime_uid}:{self.runtime_gid}',
             'Entrypoint': ['/opt/hermes/.venv/bin/python'], 'Cmd': ['-c', _BOOTSTRAP],
             'WorkingDir': '/workspace',
-            'Labels': {'hope.managed': 'true', 'hope.user_id': user['id']},
+            'Labels': {'hope.managed': 'true', 'hope.user_id': user['id'], **({'hope.webui': '62e4d7b'} if self.webui_enabled else {})},
             'Env': ['HOME=/opt/data/home', 'HERMES_HOME=/opt/data',
                     'HERMES_GATEWAY_NO_SUPERVISE=1', 'API_SERVER_ENABLED=true',
-                    'API_SERVER_HOST=0.0.0.0', 'API_SERVER_PORT=8642', f'API_SERVER_KEY={key}',
-                    f'HERMES_UPSTREAM_API_KEY={self.upstream_api_key}'],
-            'ExposedPorts': {'8642/tcp': {}},
+                    'API_SERVER_HOST=127.0.0.1', f'API_SERVER_PORT={port}', f'API_SERVER_KEY={key}',
+                    f'HERMES_UPSTREAM_API_KEY={self.upstream_api_key}'] + extra_env,
             'HostConfig': {
                 'Binds': [f'{home}:/opt/data:rw', f'{workspace}:/workspace:rw'],
-                'NetworkMode': network, 'PortBindings': {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': ''}]},
+                'NetworkMode': 'host',
                 'Init': True, 'Privileged': False, 'CapDrop': ['ALL'],
                 'SecurityOpt': ['no-new-privileges:true'], 'PidsLimit': 256,
                 'Memory': self.memory_bytes, 'MemorySwap': self.memory_bytes, 'NanoCpus': self.nano_cpus,
@@ -230,19 +365,28 @@ class Provisioner:
         env = info.get('Config', {}).get('Env') or []
         if f'API_SERVER_KEY={key}' not in env:
             raise ProvisioningError('Owned container API credential does not match private state')
-        bindings = info.get('NetworkSettings', {}).get('Ports', {}).get('8642/tcp') or []
-        if len(bindings) != 1 or bindings[0].get('HostIp') != '127.0.0.1':
-            raise ProvisioningError('Hermes API must have exactly one loopback-only publication')
-        port = str(bindings[0].get('HostPort', ''))
-        if not port.isdigit() or not 1 <= int(port) <= 65535:
-            raise ProvisioningError('Hermes API has no allocated host port')
+        if info.get('HostConfig', {}).get('NetworkMode') != 'host':
+            raise ProvisioningError('Hermes requires host networking')
+        port = self._ports().get(user['id'])
+        if not port or f'API_SERVER_HOST=127.0.0.1' not in env or f'API_SERVER_PORT={port}' not in env:
+            raise ProvisioningError('Hermes API must use its assigned loopback listener')
         return {'container_name': container_name(user), 'hermes_url': f'http://127.0.0.1:{port}', 'hermes_api_key': key}
 
     async def _healthy(self, connection):
         try:
             response = await self.health.get(connection['hermes_url'] + '/v1/models',
                 headers={'Authorization': 'Bearer ' + connection['hermes_api_key']})
-            return response.status_code == 200
+            if response.status_code != 200:
+                return False
+            if self.webui_enabled:
+                api_port = int(connection['hermes_url'].rsplit(':', 1)[1])
+                uid = next((uid for uid, port in self._ports().items() if port == api_port), None)
+                if not uid:
+                    return False
+                async with httpx.AsyncClient(verify=False, trust_env=False, timeout=3) as client:
+                    webui = await client.get(f'https://{self.webui_host}:{self._ports(True)[uid]}/health')
+                return webui.status_code == 200 and webui.json().get('status') == 'ok'
+            return True
         except httpx.HTTPError:
             return False
 
@@ -254,15 +398,30 @@ class Provisioner:
             info = await self._inspect(name)
             if info:
                 self._owned(info, user)
-            key, home, workspace = self._prepare(user, info is not None)
+            port = await self._port(user)
+            if self.webui_enabled:
+                await self._port(user, True)
+            key, home, workspace = self._prepare(user, info is not None, port)
+            # Validate credentials/spec before deleting the previous container.
+            spec = self._spec(user, key, home, workspace, port)
+            if info and (info.get('HostConfig', {}).get('NetworkMode') != 'host' or
+                    (self.webui_enabled and info.get('Config', {}).get('Labels', {}).get('hope.webui') != '62e4d7b')):
+                # Verify retained credentials before replacing a legacy container.
+                if f'API_SERVER_KEY={key}' not in (info.get('Config', {}).get('Env') or []):
+                    raise ProvisioningError('Owned container API credential does not match private state')
+                await self._request('DELETE', f'/containers/{info["Id"]}',
+                    params={'force': 'true', 'v': 'false'}, allowed=(204, 404))
+                info = None
             if info is None:
-                network = await self._network(user)
                 await self._request('POST', '/containers/create', params={'name': name},
-                    json=self._spec(user, key, home, workspace, network), allowed=(201, 409))
+                    json=spec, allowed=(201, 409))
                 info = await self._inspect(name)
                 if info is None:
                     raise ProvisioningError('Docker container creation did not persist; retry')
                 self._owned(info, user)
+            # Never start a host-networked container with an unexpected listener.
+            self._connection(user, info, key)
+            await self._remove_legacy_network(user)
             if not info.get('State', {}).get('Running'):
                 await self._request('POST', f'/containers/{info["Id"]}/start', allowed=(204, 304))
             # One overall deadline also bounds Docker inspect / health I/O; no
@@ -301,12 +460,8 @@ class Provisioner:
                 # are immutable for the lifetime of that Docker container ID.
                 await self._request('DELETE', f'/containers/{info["Id"]}',
                     params={'force': 'true', 'v': 'false'}, allowed=(204, 404))
-            name = 'hope-hermes-net-' + user['id']
-            response = await self._request('GET', f'/networks/{name}', allowed=(200, 404))
-            if response.status_code == 200:
-                network = response.json()
-                self._owned(network, user)
-                await self._request('DELETE', f'/networks/{network["Id"]}', allowed=(204, 404))
+            await self._remove_legacy_network(user)
+            # Retain port assignments alongside retained keys/data for stable recovery.
             # Intentionally retain private credential, home, workspace and memory.