feat: add machine federation

This commit is contained in:
Marc Kassubeck
2026-05-26 13:07:31 +02:00
parent b5f8810eda
commit a142f5ed40
29 changed files with 852 additions and 1150 deletions
+158
View File
@@ -0,0 +1,158 @@
import { Readable } from "node:stream";
import { WebSocket } from "ws";
import type { StoredMachine } from "./machineStore.js";
export interface MachineHttpResponse {
statusCode: number;
headers: Record<string, string | string[] | undefined>;
body?: NodeJS.ReadableStream;
}
export interface MachineJsonResponse {
statusCode: number;
headers: Record<string, string | string[] | undefined>;
body: unknown;
}
export interface MachineRequestOptions {
timeoutMs?: number;
}
export interface MachineClient {
request(method: string, path: string, body?: unknown, options?: MachineRequestOptions): Promise<MachineHttpResponse>;
requestJson(method: string, path: string, body?: unknown, options?: MachineRequestOptions): Promise<MachineJsonResponse>;
connectWebSocket(path: string): WebSocket;
}
export const DEFAULT_REMOTE_REQUEST_TIMEOUT_MS = 30_000;
export const DEFAULT_REMOTE_HEALTH_TIMEOUT_MS = 3_000;
const BLOCKED_CONFIGURED_HEADER_NAMES = new Set([
"host",
"connection",
"upgrade",
"transfer-encoding",
"content-length",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"authorization",
"cookie",
]);
export class RemoteMachineRequestError extends Error {
constructor(message: string, readonly statusCode: 502 | 504) {
super(message);
this.name = "RemoteMachineRequestError";
}
}
export class RemoteMachineClient implements MachineClient {
constructor(private readonly machine: Pick<StoredMachine, "baseUrl" | "token" | "headers">, private readonly fetchImpl: typeof fetch = fetch) {}
async request(method: string, path: string, body?: unknown, options: MachineRequestOptions = {}): Promise<MachineHttpResponse> {
const response = await this.fetchResponse(method, path, body, options);
return {
statusCode: response.status,
headers: headersToRecord(response.headers),
...(response.body === null ? {} : { body: readableFromWebResponseBody(response.body) }),
};
}
async requestJson(method: string, path: string, body?: unknown, options: MachineRequestOptions = {}): Promise<MachineJsonResponse> {
const response = await this.fetchResponse(method, path, body, options);
const text = await response.text();
const parsed: unknown = text === "" ? undefined : JSON.parse(text);
return {
statusCode: response.status,
headers: headersToRecord(response.headers),
body: parsed,
};
}
connectWebSocket(path: string): WebSocket {
const url = this.remoteUrl(path);
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
return new WebSocket(url, { headers: this.remoteHeaders() });
}
private async fetchResponse(method: string, path: string, body: unknown, options: MachineRequestOptions): Promise<Response> {
const controller = new AbortController();
const timeout = setTimeout(() => { controller.abort(); }, options.timeoutMs ?? DEFAULT_REMOTE_REQUEST_TIMEOUT_MS);
try {
const init: RequestInit = {
method,
headers: this.requestHeaders(body),
signal: controller.signal,
redirect: "manual",
};
if (body !== undefined && method !== "GET" && method !== "HEAD") init.body = JSON.stringify(body);
return await this.fetchImpl(this.remoteUrl(path), init);
} catch (error) {
if (isAbortError(error)) throw new RemoteMachineRequestError("Remote machine request timed out", 504);
throw new RemoteMachineRequestError(error instanceof Error ? error.message : String(error), 502);
} finally {
clearTimeout(timeout);
}
}
private requestHeaders(body: unknown): HeadersInit {
return {
...this.remoteHeaders(),
accept: "*/*",
...(body === undefined ? {} : { "content-type": "application/json" }),
};
}
private remoteHeaders(): Record<string, string> {
return {
...(this.machine.token === undefined || this.machine.token === "" ? {} : { authorization: `Bearer ${this.machine.token}` }),
...filterConfiguredHeaders(this.machine.headers),
};
}
private remoteUrl(path: string): URL {
const url = new URL(this.machine.baseUrl);
const separator = path.indexOf("?");
const rawPath = separator === -1 ? path : path.slice(0, separator);
const rawQuery = separator === -1 ? "" : path.slice(separator + 1);
const basePath = url.pathname.replace(/\/$/u, "");
const nextPath = rawPath.startsWith("/") ? rawPath : `/${rawPath}`;
url.pathname = `${basePath}${nextPath}`;
url.search = rawQuery === "" ? "" : `?${rawQuery}`;
url.hash = "";
return url;
}
}
export function validateConfiguredMachineHeaders(headers: Record<string, string> | undefined): Record<string, string> | undefined {
if (headers === undefined) return undefined;
return Object.fromEntries(Object.entries(headers).map(([key, value]) => {
const name = key.trim();
if (name === "") throw new Error("Machine header names must not be empty");
if (typeof value !== "string") throw new Error("Machine headers must be strings");
if (BLOCKED_CONFIGURED_HEADER_NAMES.has(name.toLowerCase())) throw new Error(`Machine header is not allowed: ${name}`);
return [name, value];
}));
}
function filterConfiguredHeaders(headers: Record<string, string> | undefined): Record<string, string> {
if (headers === undefined) return {};
return Object.fromEntries(Object.entries(headers).filter(([key]) => !BLOCKED_CONFIGURED_HEADER_NAMES.has(key.toLowerCase())));
}
function headersToRecord(headers: Headers): Record<string, string> {
return Object.fromEntries(headers.entries());
}
function readableFromWebResponseBody(body: Response["body"]): NodeJS.ReadableStream {
if (body === null) throw new Error("Response body is not readable");
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Node fetch returns a web stream that is runtime-compatible with Readable.fromWeb, but DOM and node:stream/web types are not structurally identical in this TS config.
return Readable.fromWeb(body as Parameters<typeof Readable.fromWeb>[0]);
}
function isAbortError(error: unknown): boolean {
return error instanceof DOMException && error.name === "AbortError";
}
+149
View File
@@ -0,0 +1,149 @@
import type { FastifyInstance, FastifyReply, HTTPMethods } from "fastify";
import type { WebSocket } from "ws";
import { bridgeSockets } from "../webSocketBridge.js";
import { RemoteMachineRequestError } from "./machineClient.js";
import { MachineService } from "./machineService.js";
interface HttpRouteSpec {
method: HTTPMethods;
path: string;
}
const REMOTE_HTTP_ROUTES: HttpRouteSpec[] = [
{ method: "GET", path: "/projects" },
{ method: "POST", path: "/projects" },
{ method: "DELETE", path: "/projects/:projectId" },
{ method: "GET", path: "/project-directories" },
{ method: "GET", path: "/projects/:projectId/workspaces" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/tree" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/file" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/file/preview" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/git/status" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/git/diff" },
{ method: "GET", path: "/projects/:projectId/workspaces/:workspaceId/terminals" },
{ method: "POST", path: "/projects/:projectId/workspaces/:workspaceId/terminals" },
{ method: "DELETE", path: "/projects/:projectId/workspaces/:workspaceId/terminals/:terminalId" },
{ method: "GET", path: "/files" },
{ method: "GET", path: "/activity" },
{ method: "GET", path: "/sessions" },
{ method: "POST", path: "/sessions" },
{ method: "GET", path: "/sessions/:sessionId/messages" },
{ method: "GET", path: "/sessions/:sessionId/status" },
{ method: "GET", path: "/sessions/:sessionId/models" },
{ method: "POST", path: "/sessions/:sessionId/model" },
{ method: "POST", path: "/sessions/:sessionId/model/cycle" },
{ method: "GET", path: "/sessions/:sessionId/thinking-levels" },
{ method: "POST", path: "/sessions/:sessionId/thinking-level" },
{ method: "POST", path: "/sessions/:sessionId/thinking-level/cycle" },
{ method: "GET", path: "/sessions/:sessionId/commands" },
{ method: "POST", path: "/sessions/:sessionId/prompt" },
{ method: "POST", path: "/sessions/:sessionId/shell" },
{ method: "POST", path: "/sessions/:sessionId/commands/run" },
{ method: "POST", path: "/sessions/:sessionId/commands/respond" },
{ method: "POST", path: "/sessions/:sessionId/abort" },
{ method: "POST", path: "/sessions/:sessionId/stop" },
{ method: "POST", path: "/sessions/:sessionId/archive" },
{ method: "POST", path: "/sessions/:sessionId/archive-tree" },
{ method: "POST", path: "/sessions/:sessionId/restore" },
{ method: "POST", path: "/sessions/:sessionId/detach-parent" },
{ method: "GET", path: "/auth/providers" },
{ method: "POST", path: "/auth/api-key" },
{ method: "POST", path: "/auth/logout" },
];
const REMOTE_WEBSOCKET_ROUTES = [
"/events",
"/sessions/events",
"/sessions/:sessionId/events",
"/projects/:projectId/workspaces/:workspaceId/terminals/:terminalId/socket",
];
const SAFE_RESPONSE_HEADERS = new Set([
"content-type",
"content-length",
"cache-control",
"last-modified",
"etag",
]);
export function registerMachineProxyRoutes(app: FastifyInstance, machines = new MachineService()): void {
for (const spec of REMOTE_HTTP_ROUTES) {
app.route<{ Params: { machineId: string }; Body: unknown }>({
method: spec.method,
url: `/api/machines/:machineId${spec.path}`,
handler: (request, reply) => proxyHttpRequest(machines, request.params.machineId, request.method, request.url, request.body, reply),
});
}
for (const path of REMOTE_WEBSOCKET_ROUTES) {
app.get<{ Params: { machineId: string } }>(`/api/machines/:machineId${path}`, { websocket: true }, async (socket, request) => {
await proxyWebSocket(machines, request.params.machineId, request.url, socket);
});
}
}
async function proxyHttpRequest(machines: MachineService, machineId: string, method: string, requestUrl: string, body: unknown, reply: FastifyReply): Promise<FastifyReply> {
if (machineId === "local") {
return reply.code(501).send({ error: "Local machine route is not registered for this endpoint" });
}
const client = await machines.remoteClient(machineId);
if (client === undefined) {
return reply.code(404).send({ error: "Machine not found" });
}
try {
const upstream = await client.request(method, remoteApiPath(machineId, requestUrl), body);
reply.code(upstream.statusCode);
applySafeHeaders(reply, upstream.headers);
if (upstream.body === undefined) return await reply.send();
return await reply.send(upstream.body);
} catch (error) {
return sendGatewayError(reply, machineId, error);
}
}
async function proxyWebSocket(machines: MachineService, machineId: string, requestUrl: string, socket: WebSocket): Promise<void> {
if (machineId === "local") {
socket.close(1011, "Local machine route is not registered for this endpoint");
return;
}
const client = await machines.remoteClient(machineId);
if (client === undefined) {
socket.close(1008, "Machine not found");
return;
}
try {
bridgeSockets(socket, client.connectWebSocket(remoteApiPath(machineId, requestUrl)));
} catch {
socket.close(1011, "Remote machine unavailable");
}
}
function remoteApiPath(machineId: string, requestUrl: string): string {
const machinePrefix = `/api/machines/${encodeURIComponent(machineId)}`;
const stripped = requestUrl.startsWith(machinePrefix) ? requestUrl.slice(machinePrefix.length) : requestUrl;
const compatPath = stripped.startsWith("/") ? stripped : `/${stripped}`;
return `/api${compatPath}`;
}
function applySafeHeaders(reply: FastifyReply, headers: Record<string, string | string[] | undefined>): void {
for (const [name, value] of Object.entries(headers)) {
if (value === undefined) continue;
if (!SAFE_RESPONSE_HEADERS.has(name.toLowerCase())) continue;
reply.header(name, value);
}
}
function sendGatewayError(reply: FastifyReply, machineId: string, error: unknown): FastifyReply {
const statusCode = error instanceof RemoteMachineRequestError ? error.statusCode : 502;
const label = statusCode === 504 ? "Remote machine timeout" : "Remote machine unavailable";
return reply.code(statusCode).send({
error: label,
machineId,
statusCode,
detail: error instanceof Error ? error.message : String(error),
});
}
+6
View File
@@ -12,6 +12,12 @@ export function registerMachineRoutes(app: FastifyInstance, machines = new Machi
}
});
app.get<{ Params: { machineId: string } }>("/api/machines/:machineId/health", async (request, reply) => {
const health = await machines.health(request.params.machineId);
if (health === undefined) return reply.code(404).send({ error: "Machine not found" });
return health;
});
app.get<{ Params: { machineId: string } }>("/api/machines/:machineId", async (request, reply) => {
const machine = await machines.get(request.params.machineId);
if (machine === undefined) return reply.code(404).send({ error: "Machine not found" });
@@ -43,6 +43,11 @@ describe("MachineService", () => {
await expect(service.add({ name: "Bad", baseUrl: "https://example.test/path?q=1" })).rejects.toThrow("query or hash");
});
it("rejects configured machine headers that would override proxy transport semantics", async () => {
await expect(service.add({ name: "Bad", baseUrl: "https://example.test", headers: { Authorization: "Bearer secret" } })).rejects.toThrow("not allowed");
await expect(service.add({ name: "Bad", baseUrl: "https://example.test", headers: { Connection: "close" } })).rejects.toThrow("not allowed");
});
it("does not allow local machine mutation", async () => {
await expect(service.update("local", { name: "Other" })).rejects.toThrow("Local machine cannot be changed");
await expect(service.remove("local")).rejects.toThrow("Local machine cannot be deleted");
+97 -7
View File
@@ -1,4 +1,6 @@
import type { Machine } from "../../shared/apiTypes.js";
import type { Machine, MachineHealth, PiWebComponentStatus, PiWebStatusResponse } from "../../shared/apiTypes.js";
import { getPiWebStatus } from "../piWebStatus.js";
import { DEFAULT_REMOTE_HEALTH_TIMEOUT_MS, RemoteMachineClient, type MachineClient, validateConfiguredMachineHeaders } from "./machineClient.js";
import { MachineStore, type StoredMachine } from "./machineStore.js";
export interface CreateMachineInput {
@@ -10,10 +12,20 @@ export interface CreateMachineInput {
export type UpdateMachineInput = Partial<CreateMachineInput>;
export interface MachineServiceDependencies {
localStatus?: () => Promise<PiWebStatusResponse>;
remoteClientFactory?: (machine: StoredMachine) => MachineClient;
now?: () => Date;
healthCacheTtlMs?: number;
}
const LOCAL_MACHINE_TIMESTAMP = "1970-01-01T00:00:00.000Z";
const DEFAULT_HEALTH_CACHE_TTL_MS = 5_000;
export class MachineService {
constructor(private readonly store = new MachineStore()) {}
private readonly healthCache = new Map<string, { expiresAt: number; health: MachineHealth }>();
constructor(private readonly store = new MachineStore(), private readonly deps: MachineServiceDependencies = {}) {}
async list(): Promise<Machine[]> {
return [localMachine(), ...(await this.store.list()).map(publicMachine)];
@@ -40,12 +52,69 @@ export class MachineService {
if (input.token !== undefined) patch.token = input.token;
if (input.headers !== undefined) patch.headers = validateHeaders(input.headers);
const stored = await this.store.update(id, patch);
if (stored !== undefined) this.healthCache.delete(id);
return stored === undefined ? undefined : publicMachine(stored);
}
async remove(id: string): Promise<boolean> {
if (id === "local") throw new Error("Local machine cannot be deleted");
return await this.store.remove(id);
const removed = await this.store.remove(id);
if (removed) this.healthCache.delete(id);
return removed;
}
async storedRemote(id: string): Promise<StoredMachine | undefined> {
if (id === "local") return undefined;
return (await this.store.list()).find((machine) => machine.id === id);
}
async remoteClient(id: string): Promise<MachineClient | undefined> {
const machine = await this.storedRemote(id);
return machine === undefined ? undefined : this.clientFor(machine);
}
async health(id: string): Promise<MachineHealth | undefined> {
const cached = this.healthCache.get(id);
const now = this.now().getTime();
if (cached !== undefined && cached.expiresAt > now) return cached.health;
const health = id === "local" ? await this.localHealth() : await this.remoteHealth(id);
if (health === undefined) return undefined;
this.healthCache.set(id, { expiresAt: now + (this.deps.healthCacheTtlMs ?? DEFAULT_HEALTH_CACHE_TTL_MS), health });
return health;
}
private async localHealth(): Promise<MachineHealth> {
const checkedAt = this.now().toISOString();
try {
const status = await (this.deps.localStatus ?? getPiWebStatus)();
return { machineId: "local", ok: true, checkedAt, status: "online", web: status.components.web, sessiond: status.components.sessiond };
} catch (error) {
return { machineId: "local", ok: false, checkedAt, status: "error", error: errorMessage(error) };
}
}
private async remoteHealth(id: string): Promise<MachineHealth | undefined> {
const machine = await this.storedRemote(id);
if (machine === undefined) return undefined;
const checkedAt = this.now().toISOString();
try {
const response = await this.clientFor(machine).requestJson("GET", "/api/pi-web/status", undefined, { timeoutMs: DEFAULT_REMOTE_HEALTH_TIMEOUT_MS });
if (response.statusCode >= 200 && response.statusCode < 300 && isPiWebStatusResponse(response.body)) {
return { machineId: id, ok: true, checkedAt, status: "online", web: response.body.components.web, sessiond: response.body.components.sessiond };
}
return { machineId: id, ok: false, checkedAt, status: "error", error: `Remote health returned HTTP ${String(response.statusCode)}` };
} catch (error) {
return { machineId: id, ok: false, checkedAt, status: "offline", error: errorMessage(error) };
}
}
private clientFor(machine: StoredMachine): MachineClient {
return this.deps.remoteClientFactory?.(machine) ?? new RemoteMachineClient(machine);
}
private now(): Date {
return this.deps.now?.() ?? new Date();
}
}
@@ -86,8 +155,29 @@ function optionalSecrets(input: CreateMachineInput): { token?: string; headers?:
}
function validateHeaders(value: Record<string, string>): Record<string, string> {
return Object.fromEntries(Object.entries(value).map(([key, headerValue]) => {
if (typeof headerValue !== "string") throw new Error("Machine headers must be strings");
return [key, headerValue];
}));
return validateConfiguredMachineHeaders(value) ?? {};
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function isPiWebStatusResponse(value: unknown): value is PiWebStatusResponse {
if (!isRecord(value)) return false;
const components = value["components"];
if (!isRecord(components)) return false;
return isPiWebComponentStatus(components["web"]) && isPiWebComponentStatus(components["sessiond"]);
}
function isPiWebComponentStatus(value: unknown): value is PiWebComponentStatus {
if (!isRecord(value)) return false;
const component = value["component"];
return (component === "web" || component === "sessiond")
&& typeof value["label"] === "string"
&& typeof value["stale"] === "boolean"
&& typeof value["available"] === "boolean";
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}