Archived
feat: add image attachments to the chat composer
Support pasting (Ctrl/Cmd+V), drag-and-drop, and an Attach button to add PNG/JPEG/GIF/WebP images to a message, with thumbnail previews and multi-image support. Attachments are delivered to the session using pi's native ImageContent format and are run through pi's own resizeImage so they match pi's inline image limits exactly. Image content now renders inline in the transcript. A per-message delivery toggle also lets users save attachments into the workspace `.pi-web/paste` folder and reference them so the agent reads them with its own tools. The accepted HTTP upload size is configurable via PI_WEB_MAX_UPLOAD_BYTES or the maxUploadBytes config value (default 64 MB). Closes #13
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
import { mkdtemp, readFile, readdir, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { DEFAULT_ATTACHMENT_FOLDER, saveAttachmentsToWorkspace } from "./attachmentService.js";
|
||||
|
||||
let workspace: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workspace = await mkdtemp(join(tmpdir(), "pi-web-attachments-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
||||
const pngBase64 = pngBytes.toString("base64");
|
||||
|
||||
describe("saveAttachmentsToWorkspace", () => {
|
||||
it("writes attachments into the default folder and returns relative paths", async () => {
|
||||
const fixedNow = () => new Date("2026-06-13T12:05:01.123Z");
|
||||
const saved = await saveAttachmentsToWorkspace(
|
||||
workspace,
|
||||
[
|
||||
{ kind: "image", mimeType: "image/png", data: pngBase64, name: "a.png" },
|
||||
{ kind: "image", mimeType: "image/webp", data: pngBase64, name: "b.webp" },
|
||||
],
|
||||
{ now: fixedNow },
|
||||
);
|
||||
|
||||
expect(saved).toHaveLength(2);
|
||||
expect(saved[0]?.path.startsWith(`${DEFAULT_ATTACHMENT_FOLDER}/paste-`)).toBe(true);
|
||||
expect(saved[0]?.path.endsWith(".png")).toBe(true);
|
||||
expect(saved[1]?.path.endsWith(".webp")).toBe(true);
|
||||
expect(saved[0]?.size).toBe(pngBytes.byteLength);
|
||||
|
||||
const folderEntries = await readdir(join(workspace, ".pi-web", "paste"));
|
||||
expect(folderEntries).toHaveLength(2);
|
||||
|
||||
const firstPath = saved[0]?.path ?? "";
|
||||
const written = await readFile(join(workspace, firstPath));
|
||||
expect(written.equals(pngBytes)).toBe(true);
|
||||
});
|
||||
|
||||
it("honors a custom folder", async () => {
|
||||
const saved = await saveAttachmentsToWorkspace(
|
||||
workspace,
|
||||
[{ kind: "image", mimeType: "image/png", data: pngBase64 }],
|
||||
{ folder: "uploads/images" },
|
||||
);
|
||||
expect(saved[0]?.path.startsWith("uploads/images/")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns empty for no attachments", async () => {
|
||||
expect(await saveAttachmentsToWorkspace(workspace, [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { ImageContent } from "@earendil-works/pi-ai";
|
||||
import { formatDimensionNote, resizeImage } from "@earendil-works/pi-coding-agent";
|
||||
import type { PromptAttachment, SavedPromptAttachment } from "../../shared/apiTypes.js";
|
||||
import { extensionForImageMimeType } from "../../shared/promptAttachments.js";
|
||||
import { resolveParentInsideWorkspace } from "../workspaces/pathSafety.js";
|
||||
|
||||
/**
|
||||
* Default workspace-relative folder used when saving pasted/dropped
|
||||
* attachments for the agent to read with its own tools.
|
||||
*/
|
||||
export const DEFAULT_ATTACHMENT_FOLDER = ".pi-web/paste";
|
||||
|
||||
export interface InlineImage {
|
||||
image: ImageContent;
|
||||
/** Optional human-readable dimension note produced by pi when resizing. */
|
||||
dimensionNote?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert validated attachments into pi-compatible inline image content.
|
||||
*
|
||||
* Mirrors pi's own CLI/TUI behaviour: each image is run through pi's
|
||||
* `resizeImage` so it fits within pi's max dimensions and inline byte budget
|
||||
* (2000x2000, ~4.5MB base64). Images that cannot be resized below the limit
|
||||
* are dropped, matching pi's `[Image omitted]` behaviour.
|
||||
*/
|
||||
export async function attachmentsToInlineImages(attachments: PromptAttachment[]): Promise<InlineImage[]> {
|
||||
const results: InlineImage[] = [];
|
||||
for (const attachment of attachments) {
|
||||
const bytes = Buffer.from(attachment.data, "base64");
|
||||
const resized = await resizeImage(bytes, attachment.mimeType);
|
||||
if (resized === null) continue;
|
||||
const note = formatDimensionNote(resized);
|
||||
results.push({
|
||||
image: { type: "image", data: resized.data, mimeType: resized.mimeType },
|
||||
...(note === undefined ? {} : { dimensionNote: note }),
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export interface SaveAttachmentsOptions {
|
||||
/** Workspace-relative folder to write into. Defaults to `.pi-web/paste`. */
|
||||
folder?: string;
|
||||
/** Clock injection for deterministic tests. */
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write attachments into a workspace folder and return their relative paths.
|
||||
* Filenames are collision-safe and stay inside the workspace root.
|
||||
*/
|
||||
export async function saveAttachmentsToWorkspace(
|
||||
cwd: string,
|
||||
attachments: PromptAttachment[],
|
||||
options: SaveAttachmentsOptions = {},
|
||||
): Promise<SavedPromptAttachment[]> {
|
||||
const folder = normalizeFolder(options.folder ?? DEFAULT_ATTACHMENT_FOLDER);
|
||||
const now = options.now ?? (() => new Date());
|
||||
const { target: folderTarget } = await resolveParentInsideWorkspace(cwd, folder);
|
||||
await mkdir(folderTarget, { recursive: true });
|
||||
|
||||
const stamp = timestamp(now());
|
||||
const saved: SavedPromptAttachment[] = [];
|
||||
for (const [index, attachment] of attachments.entries()) {
|
||||
const bytes = Buffer.from(attachment.data, "base64");
|
||||
const filename = `paste-${stamp}-${String(index + 1)}.${extensionForImageMimeType(attachment.mimeType)}`;
|
||||
const relativePath = `${folder}/${filename}`;
|
||||
await writeFile(join(folderTarget, filename), bytes);
|
||||
saved.push({ path: relativePath, mimeType: attachment.mimeType, size: bytes.byteLength });
|
||||
}
|
||||
return saved;
|
||||
}
|
||||
|
||||
function normalizeFolder(folder: string): string {
|
||||
return folder.split(/[\\/]+/).filter((part) => part !== "" && part !== ".").join("/");
|
||||
}
|
||||
|
||||
function timestamp(date: Date): string {
|
||||
const pad = (value: number, length = 2) => String(value).padStart(length, "0");
|
||||
return `${String(date.getFullYear())}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}-${pad(date.getMilliseconds(), 3)}`;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import type { Api, Model } from "@earendil-works/pi-ai";
|
||||
import type { Api, ImageContent, Model } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
AuthStorage,
|
||||
createAgentSessionFromServices,
|
||||
@@ -25,6 +25,10 @@ import type { AuthChange } from "./authService.js";
|
||||
import { fallbackSessionName, generateShortSessionName } from "./sessionNameGenerator.js";
|
||||
import { computeEditPreview, type EditPreviewResult } from "./editPreview.js";
|
||||
import { createPiSessionManagerGateway } from "./piSessionManagerGateway.js";
|
||||
import { attachmentsToInlineImages, saveAttachmentsToWorkspace } from "./attachmentService.js";
|
||||
import { parsePromptAttachments } from "../../shared/promptAttachments.js";
|
||||
import type { SavedPromptAttachment } from "../../shared/apiTypes.js";
|
||||
|
||||
import { cwdPathsEqual } from "../workingDirectory.js";
|
||||
import type { WorkspaceActivityService } from "../activity/workspaceActivityService.js";
|
||||
|
||||
@@ -53,6 +57,7 @@ type QueuedPromptKind = "steer" | "followUp";
|
||||
interface QueuedPrompt {
|
||||
kind: QueuedPromptKind;
|
||||
text: string;
|
||||
images?: ImageContent[];
|
||||
}
|
||||
|
||||
function requirePromptText(value: unknown): string {
|
||||
@@ -147,7 +152,7 @@ export interface PiAgentSession {
|
||||
getUserMessagesForForking(): readonly { entryId: string; text: string }[];
|
||||
getSessionStats(): { sessionId: string; totalMessages: number; userMessages: number; assistantMessages: number; toolCalls: number; tokens: ClientSessionStatus["tokens"]; cost: number };
|
||||
getContextUsage(): ClientSessionStatus["contextUsage"] | undefined;
|
||||
prompt(text: string, options?: { streamingBehavior?: "steer" | "followUp" }): Promise<void>;
|
||||
prompt(text: string, options?: { streamingBehavior?: "steer" | "followUp"; images?: ImageContent[] }): Promise<void>;
|
||||
executeBash(command: string, onChunk?: (chunk: string) => void, options?: { excludeFromContext?: boolean }): Promise<{ output: string; exitCode: number | undefined; cancelled: boolean; truncated: boolean; fullOutputPath?: string }>;
|
||||
abort(): Promise<void>;
|
||||
clearQueue(): { steering: string[]; followUp: string[] };
|
||||
@@ -407,30 +412,33 @@ export class PiSessionService {
|
||||
return commands.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
async prompt(ref: PiSessionLookup, text: unknown, streamingBehavior?: unknown): Promise<void> {
|
||||
async prompt(ref: PiSessionLookup, text: unknown, streamingBehavior?: unknown, attachments?: unknown): Promise<void> {
|
||||
const promptText = requirePromptText(text);
|
||||
const requestedBehavior = parsePromptStreamingBehavior(streamingBehavior);
|
||||
const parsedAttachments = parsePromptAttachments(attachments, { enforceInlineSizeLimit: false });
|
||||
const images = (await attachmentsToInlineImages(parsedAttachments)).map((entry) => entry.image);
|
||||
await this.assertWritable(ref);
|
||||
const session = await this.getOrOpen(ref);
|
||||
this.maybeGenerateSessionName(session, promptText);
|
||||
const isQueued = session.isStreaming || session.isCompacting;
|
||||
const behavior = isQueued ? requestedBehavior ?? "followUp" : undefined;
|
||||
if (isQueued && this.hasQueuedMessageText(session, promptText)) {
|
||||
if (isQueued && images.length === 0 && this.hasQueuedMessageText(session, promptText)) {
|
||||
this.publishActivity(session, "duplicate queued message ignored", "active");
|
||||
this.publishStatus(session);
|
||||
return;
|
||||
}
|
||||
if (session.isCompacting) {
|
||||
this.enqueuePromptDuringCompaction(session, promptText, behavior ?? "followUp");
|
||||
this.enqueuePromptDuringCompaction(session, promptText, behavior ?? "followUp", images);
|
||||
return;
|
||||
}
|
||||
void this.submitPrompt(session, promptText, behavior);
|
||||
void this.submitPrompt(session, promptText, behavior, images);
|
||||
}
|
||||
|
||||
private submitPrompt(session: PiAgentSession, text: string, behavior: QueuedPromptKind | undefined): Promise<void> {
|
||||
private submitPrompt(session: PiAgentSession, text: string, behavior: QueuedPromptKind | undefined, images: ImageContent[] = []): Promise<void> {
|
||||
this.publishActivity(session, behavior === "steer" ? "steering queued" : behavior === "followUp" ? "message queued" : "prompt accepted", "active");
|
||||
if (behavior === undefined) this.events.publish(session.sessionId, { type: "message.append", message: userTextMessage(text) });
|
||||
const promptPromise = session.prompt(text, behavior === undefined ? undefined : { streamingBehavior: behavior }).catch((error: unknown) => {
|
||||
if (behavior === undefined) this.events.publish(session.sessionId, { type: "message.append", message: userMessage(text, images) });
|
||||
const promptOptions = buildPromptOptions(behavior, images);
|
||||
const promptPromise = session.prompt(text, promptOptions).catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
this.publishActivity(session, "error", "error", message);
|
||||
this.events.publish(session.sessionId, { type: "session.error", message });
|
||||
@@ -439,14 +447,22 @@ export class PiSessionService {
|
||||
return promptPromise;
|
||||
}
|
||||
|
||||
private enqueuePromptDuringCompaction(session: PiAgentSession, text: string, kind: QueuedPromptKind): void {
|
||||
private enqueuePromptDuringCompaction(session: PiAgentSession, text: string, kind: QueuedPromptKind, images: ImageContent[] = []): void {
|
||||
const queue = this.compactionPromptQueues.get(session.sessionId) ?? [];
|
||||
queue.push({ kind, text });
|
||||
queue.push({ kind, text, ...(images.length > 0 ? { images } : {}) });
|
||||
this.compactionPromptQueues.set(session.sessionId, queue);
|
||||
this.publishActivity(session, "message queued during compaction", "active");
|
||||
this.publishStatus(session);
|
||||
}
|
||||
|
||||
async saveAttachments(ref: PiSessionLookup, attachments: unknown, folder?: string): Promise<SavedPromptAttachment[]> {
|
||||
const parsed = parsePromptAttachments(attachments, { enforceInlineSizeLimit: false });
|
||||
if (parsed.length === 0) return [];
|
||||
await this.assertWritable(ref);
|
||||
const active = await this.getActive(ref);
|
||||
return saveAttachmentsToWorkspace(active.runtime.cwd, parsed, folder === undefined ? {} : { folder });
|
||||
}
|
||||
|
||||
async shell(ref: PiSessionLookup, text: string): Promise<void> {
|
||||
await this.assertWritable(ref);
|
||||
const active = await this.getActive(ref);
|
||||
@@ -768,14 +784,14 @@ export class PiSessionService {
|
||||
const queued = this.takeCompactionPromptQueue(sessionId);
|
||||
if (queued.length === 0) return;
|
||||
this.publishStatus(session);
|
||||
for (const prompt of queued) void this.submitPrompt(session, prompt.text, prompt.kind);
|
||||
for (const prompt of queued) void this.submitPrompt(session, prompt.text, prompt.kind, prompt.images);
|
||||
return;
|
||||
}
|
||||
|
||||
const prompt = this.shiftCompactionPrompt(sessionId);
|
||||
if (prompt === undefined) return;
|
||||
this.publishStatus(session);
|
||||
const submitted = this.submitPrompt(session, prompt.text, undefined);
|
||||
const submitted = this.submitPrompt(session, prompt.text, undefined, prompt.images);
|
||||
void submitted.finally(() => { this.scheduleCompactionQueueDrain(sessionId); });
|
||||
}
|
||||
|
||||
@@ -1165,6 +1181,26 @@ function userTextMessage(text: string): { role: "user"; content: string } {
|
||||
return { role: "user", content: text };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the optimistic user message echoed to clients. When images are present
|
||||
* we mirror pi's content-array shape (`[{type:"text"}, {type:"image"}, ...]`) so
|
||||
* the local echo matches what pi persists in the session branch.
|
||||
*/
|
||||
function userMessage(text: string, images: ImageContent[]): { role: "user"; content: string | (ImageContent | { type: "text"; text: string })[] } {
|
||||
if (images.length === 0) return userTextMessage(text);
|
||||
const content: (ImageContent | { type: "text"; text: string })[] = [];
|
||||
if (text !== "") content.push({ type: "text", text });
|
||||
content.push(...images);
|
||||
return { role: "user", content };
|
||||
}
|
||||
|
||||
function buildPromptOptions(behavior: QueuedPromptKind | undefined, images: ImageContent[]): { streamingBehavior?: "steer" | "followUp"; images?: ImageContent[] } | undefined {
|
||||
const options: { streamingBehavior?: "steer" | "followUp"; images?: ImageContent[] } = {};
|
||||
if (behavior !== undefined) options.streamingBehavior = behavior;
|
||||
if (images.length > 0) options.images = images;
|
||||
return Object.keys(options).length > 0 ? options : undefined;
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
@@ -53,6 +53,28 @@ describe("session routes", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards prompt attachments and supports the save-attachments route", async () => {
|
||||
const routeApp = Fastify({ logger: false });
|
||||
await routeApp.register(fastifyWebsocket);
|
||||
const eventHub = new SessionEventHub();
|
||||
const routeService = new CapturingRouteSessionService(eventHub);
|
||||
registerSessionRoutes(routeApp, routeService, eventHub);
|
||||
|
||||
const attachments = [{ kind: "image", mimeType: "image/png", data: "QUJD", name: "shot.png" }];
|
||||
try {
|
||||
const promptResponse = await routeApp.inject({ method: "POST", url: "/sessions/session-1/prompt", payload: { text: "look", attachments } });
|
||||
expect(promptResponse.statusCode).toBe(200);
|
||||
expect(routeService.calls.at(-1)).toEqual({ lookup: "session-1", text: "look", attachments });
|
||||
|
||||
const saveResponse = await routeApp.inject({ method: "POST", url: "/sessions/session-1/attachments", payload: { attachments, folder: "uploads" } });
|
||||
expect(saveResponse.statusCode).toBe(200);
|
||||
expect(saveResponse.json()).toEqual({ attachments: [{ path: "uploads/shot.png", mimeType: "image/png", size: 3 }] });
|
||||
} finally {
|
||||
await routeService.dispose();
|
||||
await routeApp.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes cwd when per-session routes include workspace context", async () => {
|
||||
const routeApp = Fastify({ logger: false });
|
||||
await routeApp.register(fastifyWebsocket);
|
||||
@@ -98,10 +120,19 @@ class CapturingRouteSessionService extends PiSessionService {
|
||||
});
|
||||
}
|
||||
|
||||
override prompt(lookup: string | PiSessionRef, text: unknown): Promise<void> {
|
||||
this.calls.push({ lookup, text });
|
||||
override prompt(lookup: string | PiSessionRef, text: unknown, _streamingBehavior?: unknown, attachments?: unknown): Promise<void> {
|
||||
this.calls.push(attachments === undefined ? { lookup, text } : { lookup, text, attachments });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
override saveAttachments(_lookup: string | PiSessionRef, attachments: unknown, folder?: string) {
|
||||
const list = Array.isArray(attachments) ? attachments : [];
|
||||
return Promise.resolve(list.map((attachment: { mimeType: string; data: string; name?: string }) => ({
|
||||
path: `${folder ?? ".pi-web/paste"}/${attachment.name ?? "file.png"}`,
|
||||
mimeType: attachment.mimeType,
|
||||
size: Buffer.from(attachment.data, "base64").byteLength,
|
||||
})));
|
||||
}
|
||||
}
|
||||
|
||||
class RejectingSessionManager implements PiSessionManagerGateway {
|
||||
|
||||
@@ -18,6 +18,13 @@ interface PromptRequestBody {
|
||||
cwd?: unknown;
|
||||
text?: unknown;
|
||||
streamingBehavior?: unknown;
|
||||
attachments?: unknown;
|
||||
}
|
||||
|
||||
interface AttachmentsRequestBody {
|
||||
cwd?: unknown;
|
||||
attachments?: unknown;
|
||||
folder?: unknown;
|
||||
}
|
||||
|
||||
export function registerSessionRoutes(app: FastifyInstance, sessions: PiSessionService, eventHub: SessionEventHub, prefix = ""): void {
|
||||
@@ -121,13 +128,25 @@ export function registerSessionRoutes(app: FastifyInstance, sessions: PiSessionS
|
||||
app.post<{ Params: { sessionId: string }; Body: PromptRequestBody | undefined }>(`${prefix}/sessions/:sessionId/prompt`, async (request, reply) => {
|
||||
try {
|
||||
const body = optionalRecord(request.body);
|
||||
await sessions.prompt(sessionLookupFromBody(request.params.sessionId, body), body["text"], body["streamingBehavior"]);
|
||||
await sessions.prompt(sessionLookupFromBody(request.params.sessionId, body), body["text"], body["streamingBehavior"], body["attachments"]);
|
||||
return { accepted: true };
|
||||
} catch (error) {
|
||||
return reply.code(mutationErrorStatus(error)).send({ error: errorMessage(error) });
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{ Params: { sessionId: string }; Body: AttachmentsRequestBody | undefined }>(`${prefix}/sessions/:sessionId/attachments`, async (request, reply) => {
|
||||
try {
|
||||
const body = optionalRecord(request.body);
|
||||
const folder = body["folder"];
|
||||
if (folder !== undefined && typeof folder !== "string") throw new Error("folder field must be a string");
|
||||
const attachments = await sessions.saveAttachments(sessionLookupFromBody(request.params.sessionId, body), body["attachments"], folder);
|
||||
return { attachments };
|
||||
} catch (error) {
|
||||
return reply.code(mutationErrorStatus(error)).send({ error: errorMessage(error) });
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{ Params: { sessionId: string }; Body: { cwd?: unknown; text?: unknown } | undefined }>(`${prefix}/sessions/:sessionId/shell`, async (request, reply) => {
|
||||
try {
|
||||
const body = optionalRecord(request.body);
|
||||
|
||||
Reference in New Issue
Block a user