314 lines
16 KiB
Python
314 lines
16 KiB
Python
"""Trusted Docker-UDS control plane for independent Hermes accounts.
|
|
|
|
Only this service may receive the host Docker socket. Bind-source paths must be
|
|
identical in the control service and Docker host (mount the runtime root there).
|
|
The official image's root/s6 boot is deliberately bypassed: a nonroot Python
|
|
bootstrap seeds bundled skills, then execs the verified .venv Hermes CLI. Image
|
|
compatibility still needs deployment qualification before an image pin is promoted.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import tempfile
|
|
|
|
import httpx
|
|
|
|
|
|
class ProvisioningError(RuntimeError):
|
|
"""Safe-to-display provisioning failure (never includes Docker env/secrets)."""
|
|
|
|
|
|
class OwnershipError(ProvisioningError):
|
|
"""An existing resource is not owned by this account."""
|
|
|
|
|
|
# Run as the image's hermes UID without executing root-only s6 startup. No source
|
|
# checkout/profile is copied: only the image's public bundled skills are seeded.
|
|
_BOOTSTRAP = """
|
|
import os, pathlib, shutil
|
|
home = pathlib.Path('/opt/data')
|
|
for name in ('home', 'sessions', 'memories', 'skills', 'cron', 'hooks', 'logs', 'cache'):
|
|
(home / name).mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
source = pathlib.Path('/opt/hermes/skills')
|
|
if source.is_dir() and not (home / '.hope-skills-seeded').exists():
|
|
shutil.copytree(source, home / 'skills', dirs_exist_ok=True)
|
|
(home / '.hope-skills-seeded').touch(mode=0o600)
|
|
os.makedirs('/tmp/hermes-runtime', mode=0o700, exist_ok=True)
|
|
os.execv('/opt/hermes/.venv/bin/hermes', ['hermes', 'gateway', 'run'])
|
|
"""
|
|
|
|
|
|
def container_name(user: dict) -> str:
|
|
uid, username = user.get('id'), user.get('username')
|
|
if not isinstance(uid, str) or not re.fullmatch(r'[0-9a-f]{32}', uid):
|
|
raise ValueError('Account id must be a lowercase UUID hex string')
|
|
if not isinstance(username, str) or not re.fullmatch(r'[a-z][a-z0-9]*(?:-[a-z0-9]+)*', username) or len(username) > 48:
|
|
raise ValueError('Account username must be a lowercase slug')
|
|
return f'hope-hermes-{username}-{uid[:8]}'
|
|
|
|
|
|
class Provisioner:
|
|
def __init__(self, state_dir=None, *, docker_socket=None, image=None,
|
|
upstream_api_key=None, docker_client=None, health_client=None,
|
|
ready_timeout=60, poll_interval=.5, runtime_uid=10000,
|
|
runtime_gid=10000, memory_bytes=2 * 1024 ** 3, nano_cpus=2_000_000_000):
|
|
self.state_dir = Path(state_dir or os.environ.get('HOPE_HERMES_DATA_DIR', '.runtime/hermes')).absolute()
|
|
self.image = image or os.environ.get('HERMES_IMAGE', 'nousresearch/hermes-agent@sha256:d4da4a40cd7a28aba983775d9fd31d94cbf153eeb0cb9e844d6d0f612b7c24db')
|
|
self.upstream_api_key = upstream_api_key if upstream_api_key is not None else os.environ.get('HERMES_UPSTREAM_API_KEY', '')
|
|
self.ready_timeout, self.poll_interval = ready_timeout, poll_interval
|
|
self.runtime_uid, self.runtime_gid = runtime_uid, runtime_gid
|
|
if runtime_uid <= 0 or runtime_gid <= 0:
|
|
raise ValueError('Hermes must run nonroot')
|
|
self.memory_bytes, self.nano_cpus = memory_bytes, nano_cpus
|
|
self._own_docker, self._own_health = docker_client is None, health_client is None
|
|
# No TCP Docker endpoint, DOCKER_HOST, subprocess, or proxy inheritance.
|
|
self.docker = docker_client or httpx.AsyncClient(
|
|
transport=httpx.AsyncHTTPTransport(uds=docker_socket or os.environ.get('DOCKER_SOCKET', '/var/run/docker.sock')),
|
|
base_url='http://docker', timeout=20, trust_env=False)
|
|
self.health = health_client or httpx.AsyncClient(timeout=3, trust_env=False, follow_redirects=False)
|
|
self._locks = {}
|
|
|
|
async def aclose(self):
|
|
if self._own_docker:
|
|
await self.docker.aclose()
|
|
if self._own_health:
|
|
await self.health.aclose()
|
|
|
|
def _lock(self, user):
|
|
container_name(user)
|
|
return self._locks.setdefault(user['id'], asyncio.Lock())
|
|
|
|
async def _request(self, method, path, *, allowed=(200,), **kwargs):
|
|
try:
|
|
response = await self.docker.request(method, path, **kwargs)
|
|
except httpx.HTTPError as exc:
|
|
raise ProvisioningError('Docker control socket unavailable; retry provisioning') from exc
|
|
if response.status_code not in allowed:
|
|
raise ProvisioningError(f'Docker operation failed (HTTP {response.status_code}); retry provisioning')
|
|
return response
|
|
|
|
async def _inspect(self, name):
|
|
response = await self._request('GET', f'/containers/{name}/json', allowed=(200, 404))
|
|
return response.json() if response.status_code == 200 else None
|
|
|
|
@staticmethod
|
|
def _owned(resource, user):
|
|
labels = resource.get('Config', {}).get('Labels') or resource.get('Labels') or {}
|
|
if labels.get('hope.managed') != 'true' or labels.get('hope.user_id') != user['id']:
|
|
raise OwnershipError('Refusing resource whose Hope ownership labels do not match')
|
|
|
|
@staticmethod
|
|
def _directory(path):
|
|
# Do not follow a user-created symlink into another account's state.
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime directory cannot be a symlink')
|
|
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
os.chmod(path, 0o700)
|
|
|
|
@staticmethod
|
|
def _write_private(path, value):
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime file cannot be a symlink')
|
|
fd, temp = tempfile.mkstemp(prefix='.hope-', dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'w') as stream:
|
|
stream.write(value)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temp, path)
|
|
finally:
|
|
if os.path.exists(temp):
|
|
temporary = Path(temp)
|
|
if temporary.resolve().parent != path.parent.resolve() or not temporary.name.startswith('.hope-'):
|
|
raise ProvisioningError('Refusing unexpected temporary-file cleanup path')
|
|
temporary.unlink()
|
|
|
|
def _paths(self, user):
|
|
container_name(user)
|
|
# Every ancestor under the configured root is control-owned, not mounted
|
|
# into Hermes. HOME cannot access the sibling API credential file.
|
|
return self.state_dir / user['id'], self.state_dir / user['id'] / 'home', self.state_dir / user['id'] / 'workspace'
|
|
|
|
def _read_key(self, user):
|
|
private, _, _ = self._paths(user)
|
|
path = private / 'api-key'
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime file cannot be a symlink')
|
|
try:
|
|
key = path.read_text().strip()
|
|
except FileNotFoundError:
|
|
return None
|
|
if len(key) < 32 or '\n' in key:
|
|
raise ProvisioningError('Invalid persisted Hermes API credential')
|
|
return key
|
|
|
|
def _prepare(self, user, exists):
|
|
private, home, workspace = self._paths(user)
|
|
self._directory(self.state_dir)
|
|
for path in (private, home, workspace):
|
|
self._directory(path)
|
|
key = self._read_key(user)
|
|
if not key:
|
|
if exists:
|
|
raise ProvisioningError('Owned container has no persisted API credential; refusing unsafe adoption')
|
|
key = secrets.token_urlsafe(48)
|
|
self._write_private(private / 'api-key', key + '\n')
|
|
config = home / 'config.yaml'
|
|
if not config.exists():
|
|
# JSON is a YAML subset; no credentials are placed in configuration.
|
|
self._write_private(config, json.dumps(self.config(), indent=2) + '\n')
|
|
elif config.is_symlink():
|
|
raise ProvisioningError('Hermes config cannot be a symlink')
|
|
if os.geteuid() == 0:
|
|
# Only seeded directory roots/config, never recursive retained data.
|
|
for path in (home, workspace, config):
|
|
os.chown(path, self.runtime_uid, self.runtime_gid)
|
|
elif home.stat().st_uid != self.runtime_uid:
|
|
raise ProvisioningError('Control service must prepare private directories for the Hermes runtime UID')
|
|
return key, home, workspace
|
|
|
|
@staticmethod
|
|
def config():
|
|
model = os.environ.get('HERMES_UPSTREAM_MODEL', 'auto')
|
|
return {
|
|
'model': {'default': model, 'provider': 'custom:hope-upstream'},
|
|
'providers': {'hope-upstream': {'api': os.environ.get('HERMES_UPSTREAM_URL', 'http://192.168.86.5:2099/v1'),
|
|
'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': model,
|
|
'models': [model], 'discover_models': False}},
|
|
'agent': {'reasoning_effort': 'none'},
|
|
'terminal': {'backend': 'local', 'cwd': '/workspace'},
|
|
'platform_toolsets': {'api_server': ['terminal', 'file', 'web', 'browser', 'memory', 'skills', 'delegation']},
|
|
'gateway': {'multiplex_profiles': False,
|
|
'api_server': {'enabled': True, 'host': '0.0.0.0', 'port': 8642,
|
|
'max_concurrent_runs': 1, 'tool_progress_events': True}},
|
|
}
|
|
|
|
async def _network(self, user):
|
|
name = 'hope-hermes-net-' + user['id']
|
|
response = await self._request('GET', f'/networks/{name}', allowed=(200, 404))
|
|
if response.status_code == 404:
|
|
await self._request('POST', '/networks/create', allowed=(201, 409), json={
|
|
'Name': name, 'Driver': 'bridge', 'CheckDuplicate': True,
|
|
'Labels': {'hope.managed': 'true', 'hope.user_id': user['id']}})
|
|
response = await self._request('GET', f'/networks/{name}')
|
|
network = response.json()
|
|
self._owned(network, user)
|
|
if network.get('Driver') != 'bridge' or network.get('Internal'):
|
|
raise ProvisioningError('Owned Hermes network is not an outbound-capable dedicated bridge')
|
|
return name
|
|
|
|
def _spec(self, user, key, home, workspace, network):
|
|
return {
|
|
'Image': self.image, 'User': f'{self.runtime_uid}:{self.runtime_gid}',
|
|
'Entrypoint': ['/opt/hermes/.venv/bin/python'], 'Cmd': ['-c', _BOOTSTRAP],
|
|
'WorkingDir': '/workspace',
|
|
'Labels': {'hope.managed': 'true', 'hope.user_id': user['id']},
|
|
'Env': ['HOME=/opt/data/home', 'HERMES_HOME=/opt/data',
|
|
'HERMES_GATEWAY_NO_SUPERVISE=1', 'API_SERVER_ENABLED=true',
|
|
'API_SERVER_HOST=0.0.0.0', 'API_SERVER_PORT=8642', f'API_SERVER_KEY={key}',
|
|
f'HERMES_UPSTREAM_API_KEY={self.upstream_api_key}'],
|
|
'ExposedPorts': {'8642/tcp': {}},
|
|
'HostConfig': {
|
|
'Binds': [f'{home}:/opt/data:rw', f'{workspace}:/workspace:rw'],
|
|
'NetworkMode': network, 'PortBindings': {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': ''}]},
|
|
'Init': True, 'Privileged': False, 'CapDrop': ['ALL'],
|
|
'SecurityOpt': ['no-new-privileges:true'], 'PidsLimit': 256,
|
|
'Memory': self.memory_bytes, 'MemorySwap': self.memory_bytes, 'NanoCpus': self.nano_cpus,
|
|
'ShmSize': 256 * 1024 ** 2,
|
|
'RestartPolicy': {'Name': 'unless-stopped'},
|
|
'LogConfig': {'Type': 'json-file', 'Config': {'max-size': '10m', 'max-file': '3'}},
|
|
},
|
|
}
|
|
|
|
def _connection(self, user, info, key):
|
|
self._owned(info, user)
|
|
env = info.get('Config', {}).get('Env') or []
|
|
if f'API_SERVER_KEY={key}' not in env:
|
|
raise ProvisioningError('Owned container API credential does not match private state')
|
|
bindings = info.get('NetworkSettings', {}).get('Ports', {}).get('8642/tcp') or []
|
|
if len(bindings) != 1 or bindings[0].get('HostIp') != '127.0.0.1':
|
|
raise ProvisioningError('Hermes API must have exactly one loopback-only publication')
|
|
port = str(bindings[0].get('HostPort', ''))
|
|
if not port.isdigit() or not 1 <= int(port) <= 65535:
|
|
raise ProvisioningError('Hermes API has no allocated host port')
|
|
return {'container_name': container_name(user), 'hermes_url': f'http://127.0.0.1:{port}', 'hermes_api_key': key}
|
|
|
|
async def _healthy(self, connection):
|
|
try:
|
|
response = await self.health.get(connection['hermes_url'] + '/v1/models',
|
|
headers={'Authorization': 'Bearer ' + connection['hermes_api_key']})
|
|
return response.status_code == 200
|
|
except httpx.HTTPError:
|
|
return False
|
|
|
|
async def ensure(self, user: dict) -> dict:
|
|
async with self._lock(user):
|
|
if not self.upstream_api_key:
|
|
raise ProvisioningError('HERMES_UPSTREAM_API_KEY must be provided at runtime')
|
|
name = container_name(user)
|
|
info = await self._inspect(name)
|
|
if info:
|
|
self._owned(info, user)
|
|
key, home, workspace = self._prepare(user, info is not None)
|
|
if info is None:
|
|
network = await self._network(user)
|
|
await self._request('POST', '/containers/create', params={'name': name},
|
|
json=self._spec(user, key, home, workspace, network), allowed=(201, 409))
|
|
info = await self._inspect(name)
|
|
if info is None:
|
|
raise ProvisioningError('Docker container creation did not persist; retry')
|
|
self._owned(info, user)
|
|
if not info.get('State', {}).get('Running'):
|
|
await self._request('POST', f'/containers/{info["Id"]}/start', allowed=(204, 304))
|
|
# One overall deadline also bounds Docker inspect / health I/O; no
|
|
# deletion on failed health, so a retry starts/adopts the same state.
|
|
try:
|
|
async with asyncio.timeout(self.ready_timeout):
|
|
while True:
|
|
info = await self._inspect(name)
|
|
if info is None:
|
|
raise ProvisioningError('Hermes container disappeared; retry')
|
|
self._owned(info, user)
|
|
if info.get('State', {}).get('Running'):
|
|
connection = self._connection(user, info, key)
|
|
if await self._healthy(connection):
|
|
return connection
|
|
await asyncio.sleep(self.poll_interval)
|
|
except TimeoutError as exc:
|
|
raise ProvisioningError('Hermes API readiness timed out; retained container and data for retry') from exc
|
|
|
|
async def ready(self, user: dict) -> bool:
|
|
try:
|
|
info = await self._inspect(container_name(user))
|
|
if not info or not info.get('State', {}).get('Running'):
|
|
return False
|
|
key = self._read_key(user)
|
|
return bool(key) and await self._healthy(self._connection(user, info, key))
|
|
except (ProvisioningError, ValueError, OSError):
|
|
return False
|
|
|
|
async def remove(self, user: dict):
|
|
async with self._lock(user):
|
|
info = await self._inspect(container_name(user))
|
|
if info:
|
|
self._owned(info, user)
|
|
# Operate on inspected immutable ID, never a raced name. Labels
|
|
# are immutable for the lifetime of that Docker container ID.
|
|
await self._request('DELETE', f'/containers/{info["Id"]}',
|
|
params={'force': 'true', 'v': 'false'}, allowed=(204, 404))
|
|
name = 'hope-hermes-net-' + user['id']
|
|
response = await self._request('GET', f'/networks/{name}', allowed=(200, 404))
|
|
if response.status_code == 200:
|
|
network = response.json()
|
|
self._owned(network, user)
|
|
await self._request('DELETE', f'/networks/{network["Id"]}', allowed=(204, 404))
|
|
# Intentionally retain private credential, home, workspace and memory.
|
|
|
|
|
|
DockerProvisioner = Provisioner
|