diff --git a/.changeset/honor-archive-data-dir.md b/.changeset/honor-archive-data-dir.md index df07440..f50f6c5 100644 --- a/.changeset/honor-archive-data-dir.md +++ b/.changeset/honor-archive-data-dir.md @@ -2,6 +2,6 @@ "@jmfederico/pi-web": patch --- -Store session archive metadata and archived session files under `PI_WEB_DATA_DIR` when configured. +Store session archive metadata and archived session files under `PI_WEB_DATA_DIR` when configured, and automatically migrate a legacy archive on the first eligible session-daemon startup after upgrading. -Previously, session archives always used `~/.pi-web`, even when `PI_WEB_DATA_DIR` selected another managed-state root. Existing archives created with a custom `PI_WEB_DATA_DIR` remain in `~/.pi-web` and are not migrated automatically. Before upgrading, stop the session daemon and back up both locations before reconciling them manually. Because the archive index stores absolute `archivePath` values, update those values when moving archived files. +Migration runs only when `PI_WEB_DATA_DIR` explicitly selects a different root, the legacy index and every referenced file form a complete valid archive, and the destination archive is pristine. PI WEB copies and verifies files across filesystem boundaries, rewrites their `archivePath` values, atomically commits the destination index, and only then removes legacy archive state. Ambiguous, invalid, partial, or coexisting layouts are left untouched instead of being merged or overwritten; active Pi session files are never moved. diff --git a/docs/config.md b/docs/config.md index e171afe..2f0e398 100644 --- a/docs/config.md +++ b/docs/config.md @@ -122,6 +122,26 @@ Rows with JSON key `—` are runtime-only environment variables, not config-file ## Key details +### Managed data directory and session archive migration + +`PI_WEB_DATA_DIR` selects the root for PI WEB-managed state, including the session archive index and archived session files. It does not move active Pi session files from `PI_CODING_AGENT_SESSION_DIR`. + +After an upgrade from a version that stored session archives only in `~/.pi-web`, the session daemon performs a one-time migration on the first eligible startup, before it creates the session runtime, registers routes, or starts listening. Automatic migration is deliberately strict. It runs only when all of these conditions are proven: + +1. `PI_WEB_DATA_DIR` is explicitly set to a non-empty value and resolves to a root other than `~/.pi-web`. +2. `~/.pi-web/archived-sessions.json` is a regular file that parses with the current archive schema. +3. `$PI_WEB_DATA_DIR/archived-sessions.json` does not exist. +4. `$PI_WEB_DATA_DIR/archived-sessions/` is absent or is a real, empty directory. +5. Every defined `archivePath` points directly into the real `~/.pi-web/archived-sessions/` directory, names an existing regular file, and maps uniquely to the destination. Metadata-only records without an `archivePath` are preserved. +6. The legacy archive directory is absent or empty when no files are referenced; otherwise it contains exactly the regular files referenced by the index, with no extra files, symlinks, nested directories, or temporary entries. +7. Session IDs are non-empty and unique, source paths are unique under the host's path semantics, and destination mappings have no case/normalization collisions or other ambiguity. + +An eligible migration copies and byte-verifies the archived files, so it works when the legacy and configured roots are on different filesystems. It rewrites only the migrated `archivePath` values, preserves the remaining archive metadata, verifies the complete destination, and atomically publishes the destination index. Only after that commit does it remove the legacy archived files, archive directory, and index. + +If any preflight condition is false or filesystem inspection is inconclusive, migration makes no archive filesystem changes and the session daemon starts against the configured destination. PI WEB does not infer intent, merge indexes, adopt partial files, or overwrite either side. If you expected a migration but it was skipped, stop the daemon, back up both roots, and inspect them before reconciling the state manually; do not delete either side based only on the skip. + +A copy, verification, or commit failure before the destination index is published keeps the legacy index authoritative, rolls back artifacts owned by that attempt where possible, and stops session-daemon startup. Correct the reported filesystem problem and remove only artifacts you have verified belong to the failed attempt before restarting. If cleanup fails after the destination index is committed, the daemon warns and starts from the verified destination; treat that destination as authoritative and inspect any legacy or staging leftovers manually rather than merging them back. + ### External path access `pathAccess.allowedPaths` grants PI WEB's file explorer and absolute `@` path completions access to specific filesystem roots outside the current workspace. diff --git a/src/server/sessions/sessionArchiveMigration.test.ts b/src/server/sessions/sessionArchiveMigration.test.ts index 6f1b6cf..876534c 100644 --- a/src/server/sessions/sessionArchiveMigration.test.ts +++ b/src/server/sessions/sessionArchiveMigration.test.ts @@ -3,17 +3,19 @@ import { access, appendFile, copyFile, + link as createHardLink, lstat, mkdtemp, mkdir, readFile, readdir, + realpath, rm, unlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, sep } from "node:path"; +import { dirname, join, resolve, sep } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { inspectLegacySessionArchiveMigration, @@ -122,16 +124,15 @@ describe("legacy session archive migration preflight", () => { await expectLegacyStateUntouched(fixture); }); - it("skips a non-empty destination archive directory without changing either side", async () => { + it("leaves an interrupted pre-commit file untouched instead of adopting a non-empty destination archive", async () => { const fixture = await createLegacyArchiveFixture({ createDestinationArchive: true }); - const destinationEntry = join(fixture.destinationArchiveDir, "already-here.jsonl"); - await writeFile(destinationEntry, "destination owner\n", "utf8"); + await writeFile(fixture.destinationFilePath, "partial destination copy\n", "utf8"); await expect(migrateLegacySessionArchive(fixture.options)).resolves.toEqual({ status: "skipped", reason: "destination-archive-not-empty-or-invalid", }); - await expect(readFile(destinationEntry, "utf8")).resolves.toBe("destination owner\n"); + await expect(readFile(fixture.destinationFilePath, "utf8")).resolves.toBe("partial destination copy\n"); await expectLegacyStateUntouched(fixture); }); @@ -155,6 +156,42 @@ describe("legacy session archive migration preflight", () => { expect(await exists(fixture.destinationRoot)).toBe(false); }); + it("uses Windows case-insensitive path semantics without weakening Linux containment", async () => { + const fixture = await createLegacyArchiveFixture(); + const casedArchivePath = resolve(fixture.legacyFilePath.toUpperCase()); + const firstRecord = requiredRecord(fixture.document.sessions, 0); + await writeArchiveIndex(fixture.legacyIndexPath, { + ...fixture.document, + sessions: [ + { ...firstRecord, archivePath: casedArchivePath }, + ...fixture.document.sessions.slice(1), + ], + }); + const mapCaseVariant = (path: string): string => path.toLowerCase() === casedArchivePath.toLowerCase() + ? fixture.legacyFilePath + : path; + const fileSystem = { + lstat: (path: string) => lstat(mapCaseVariant(path)), + realpath: (path: string) => realpath(mapCaseVariant(path)), + }; + + await expect(inspectLegacySessionArchiveMigration({ + ...fixture.options, + platform: "linux", + fileSystem, + })).resolves.toEqual({ status: "skipped", reason: "legacy-archive-layout-invalid" }); + await expect(inspectLegacySessionArchiveMigration({ + ...fixture.options, + platform: "win32", + fileSystem, + })).resolves.toEqual({ + status: "eligible", + legacyIndexPath: fixture.legacyIndexPath, + destinationIndexPath: fixture.destinationIndexPath, + archiveFileCount: 1, + }); + }); + it("skips legacy directories containing unindexed entries", async () => { const fixture = await createLegacyArchiveFixture(); const unexpectedPath = join(fixture.legacyArchiveDir, "unexpected.tmp"); @@ -211,9 +248,10 @@ describe("legacy session archive migration execution", () => { await Promise.all(tempRoots.splice(0).map((path) => rm(path, { recursive: true, force: true }))); }); - it("stages and verifies copies, atomically publishes the rewritten index, then removes legacy state", async () => { + it("copies across the source boundary, atomically publishes the rewritten index, then removes legacy state", async () => { const fixture = await createLegacyArchiveFixture({ createDestinationArchive: true }); const copies: { source: string; destination: string; mode: number }[] = []; + const links: { source: string; destination: string }[] = []; await expect(migrateLegacySessionArchive({ ...fixture.options, @@ -222,6 +260,16 @@ describe("legacy session archive migration execution", () => { copies.push({ source, destination, mode }); await copyFile(source, destination, mode); }, + link: async (source, destination) => { + links.push({ source, destination }); + await createHardLink(source, destination); + }, + unlink: async (path) => { + if (path === fixture.legacyFilePath || path === fixture.legacyIndexPath) { + expect(await exists(fixture.destinationIndexPath)).toBe(true); + } + await unlink(path); + }, }, })).resolves.toEqual({ status: "migrated", archiveFileCount: 1, cleanup: "complete" }); @@ -229,6 +277,10 @@ describe("legacy session archive migration execution", () => { expect(copies[0]).toMatchObject({ source: fixture.legacyFilePath, mode: constants.COPYFILE_EXCL }); expect(copies[0]?.destination).toContain(".archived-sessions-migration-test-attempt"); expect(copies[1]).toMatchObject({ destination: fixture.destinationFilePath, mode: constants.COPYFILE_EXCL }); + expect(links).toEqual([{ + source: join(fixture.destinationRoot, ".archived-sessions-migration-test-attempt", "archived-sessions.json"), + destination: fixture.destinationIndexPath, + }]); await expect(readFile(fixture.destinationFilePath, "utf8")).resolves.toBe("legacy session\n"); const migratedDocument: unknown = JSON.parse(await readFile(fixture.destinationIndexPath, "utf8")); @@ -250,6 +302,29 @@ describe("legacy session archive migration execution", () => { ])); }); + it("retries safely when an interrupted staging-only attempt left an unowned sibling tree", async () => { + const fixture = await createLegacyArchiveFixture(); + const abandonedFile = join( + fixture.destinationRoot, + ".archived-sessions-migration-interrupted-attempt", + "files", + "abandoned.jsonl", + ); + await mkdir(dirname(abandonedFile), { recursive: true }); + await writeFile(abandonedFile, "unowned staging data\n", "utf8"); + + await expect(migrateLegacySessionArchive(fixture.options)).resolves.toEqual({ + status: "migrated", + archiveFileCount: 1, + cleanup: "complete", + }); + + await expect(readFile(abandonedFile, "utf8")).resolves.toBe("unowned staging data\n"); + await expect(readFile(fixture.destinationFilePath, "utf8")).resolves.toBe("legacy session\n"); + expect(await exists(fixture.destinationIndexPath)).toBe(true); + expect(await exists(fixture.legacyIndexPath)).toBe(false); + }); + it("rolls back destination artifacts and preserves all legacy state when staged-copy verification fails", async () => { const fixture = await createLegacyArchiveFixture(); @@ -269,6 +344,31 @@ describe("legacy session archive migration execution", () => { await expect(readdir(fixture.destinationRoot)).resolves.toEqual([]); }); + it("revalidates source files before commit and rolls back if one changes during migration", async () => { + const fixture = await createLegacyArchiveFixture(); + + const result = await migrateLegacySessionArchive({ + ...fixture.options, + fileSystem: { + copyFile: async (source, destination, mode) => { + await copyFile(source, destination, mode); + if (destination === fixture.destinationFilePath) { + await appendFile(fixture.legacyFilePath, "changed during migration\n", "utf8"); + } + }, + }, + }); + + expect(result).toMatchObject({ status: "failed", phase: "commit-index", rollbackErrors: [] }); + await expect(readFile(fixture.legacyIndexPath, "utf8")).resolves.toBe(fixture.sourceIndexContents); + await expect(readFile(fixture.legacyFilePath, "utf8")).resolves.toBe( + "legacy session\nchanged during migration\n", + ); + await expect(readFile(fixture.activeFilePath, "utf8")).resolves.toBe("active session\n"); + expect(await exists(fixture.destinationIndexPath)).toBe(false); + expect(await exists(fixture.destinationArchiveDir)).toBe(false); + }); + it("rolls back published files but never source state when atomic index publication fails", async () => { const fixture = await createLegacyArchiveFixture(); const publicationError = Object.assign(new Error("link failed"), { code: "EIO" }); @@ -292,6 +392,31 @@ describe("legacy session archive migration execution", () => { await expect(readdir(fixture.destinationRoot)).resolves.toEqual([]); }); + it("does not overwrite a destination index that appears at the atomic commit boundary", async () => { + const fixture = await createLegacyArchiveFixture(); + const publicationError = Object.assign(new Error("destination index won the race"), { code: "EEXIST" }); + + const result = await migrateLegacySessionArchive({ + ...fixture.options, + fileSystem: { + link: async (_source, destination) => { + await writeFile(destination, "destination owner\n", { encoding: "utf8", flag: "wx" }); + throw publicationError; + }, + }, + }); + + expect(result).toMatchObject({ + status: "failed", + phase: "commit-index", + error: publicationError, + rollbackErrors: [], + }); + await expect(readFile(fixture.destinationIndexPath, "utf8")).resolves.toBe("destination owner\n"); + expect(await exists(fixture.destinationArchiveDir)).toBe(false); + await expectLegacyStateUntouched(fixture); + }); + it("keeps the committed destination authoritative when legacy cleanup fails", async () => { const fixture = await createLegacyArchiveFixture(); const cleanupError = Object.assign(new Error("source cleanup failed"), { code: "EACCES" });