diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index e21a189..0f989d0 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -36,12 +36,22 @@ WORKDIR /workspace COPY package.json package-lock.json ./ COPY scripts/install-git-hooks.mjs scripts/install-git-hooks.mjs +# Keep an immutable dependency seed outside /workspace, which is hidden by the +# checkout bind mount at runtime. A cached generation is added after custom +# image hooks so it identifies the final dependency tree. RUN npm ci \ - && ln -sf /workspace/node_modules/.bin/pi /usr/local/bin/pi \ + && install -d -m 0755 /opt/pi-web-dev-dependencies \ + && cp package.json package-lock.json /opt/pi-web-dev-dependencies/ \ + && chmod -R a+rwX /workspace/node_modules \ + && mv /workspace/node_modules /opt/pi-web-dev-dependencies/node_modules \ + && ln -s /opt/pi-web-dev-dependencies/node_modules /workspace/node_modules \ + && ln -sf /opt/pi-web-dev-dependencies/node_modules/.bin/pi /usr/local/bin/pi \ && npm cache clean --force \ - && chmod -R a+rwX /workspace/node_modules /data \ + && chmod -R a+rwX /data \ && chmod 0777 /workspace +COPY --chmod=0755 docker/internal/dev/sync-node-modules /usr/local/sbin/pi-web-dev-sync-node-modules + COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins COPY docker/internal/bin/hostexec /usr/local/bin/hostexec @@ -49,6 +59,8 @@ COPY docker/pi-web-docker /usr/local/bin/pi-web-docker RUN chmod 0755 /usr/local/bin/hostexec /usr/local/bin/pi-web-docker COPY docker/custom-image.d/ /tmp/pi-web-custom-image.d/ +# Image hooks use the temporary /workspace/node_modules symlink. Leave an empty +# directory afterward so Compose can mount and populate the dependency volume. RUN bash -euxo pipefail -c '\ shopt -s nullglob; \ for script in /tmp/pi-web-custom-image.d/*.sh; do \ @@ -56,9 +68,15 @@ RUN bash -euxo pipefail -c '\ bash "${script}"; \ done; \ rm -rf /tmp/pi-web-custom-image.d; \ + test -L /workspace/node_modules; \ + rm /workspace/node_modules; \ + install -d -m 0777 /workspace/node_modules; \ zypper clean --all; \ rm -rf /var/cache/zypp/* \ ' +# Cache the generation with the completed seed. Changes to any preceding layer, +# including custom image hooks, rerun this step and refresh the named volume. +RUN node -e 'process.stdout.write(`${require("node:crypto").randomUUID()}\n`)' > /opt/pi-web-dev-dependencies/generation EXPOSE 8504 8505 diff --git a/docker/README.md b/docker/README.md index 9f72d59..ff5bda8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -327,13 +327,9 @@ Use this shared directory to switch between runtime and dev mode, not to run bot For sessions to appear under the same workspace in both modes, use the same project path in PI WEB. On Linux, prefer host-mounted paths such as `/home/core/`, `/srv/`, or `/opt/`. On Mac, prefer paths under `/Users//...`. The dev container also exposes this checkout as `/workspace` so the PI WEB dev server can run from it, but sessions started against `/workspace` are organized under that different working-directory path and will not line up with runtime sessions for the host-mounted path. -When `package-lock.json` changes, rebuild the dev image and recreate the `node_modules` volume so the bind-mounted checkout sees the new dependency tree: +Development startup keeps the persistent `node_modules` volume synchronized with the dependency tree built into the dev image. When `package.json`, `package-lock.json`, the Node image, or another dependency-build input changes, `start` or `update` rebuilds the image and `data-init` refreshes the volume before `sessiond` starts. Manual volume removal is not required. -```bash -./docker/pi-web-docker --dev stop -docker volume rm pi-web-dev_node_modules -./docker/pi-web-docker --dev start -``` +If Compose is invoked directly without rebuilding after a manifest change, `data-init` stops with a mismatch message instead of starting against stale dependencies. Run `./docker/pi-web-docker --dev start` or `./docker/pi-web-docker --dev update` to rebuild and synchronize it. ## Local checkout validation diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml index cbf76dc..27cac48 100644 --- a/docker/compose.dev.yml +++ b/docker/compose.dev.yml @@ -56,14 +56,14 @@ services: set -euo pipefail mkdir -p /data/home /data/config /data/npm-cache /data/pi-web /data/pi-agent chown -R "${PI_WEB_UID:-1000}:${PI_WEB_GID:-1000}" /data + /usr/local/sbin/pi-web-dev-sync-node-modules user: "0:0" security_opt: - label=disable environment: PI_WEB_UID: ${PI_WEB_UID:-1000} PI_WEB_GID: ${PI_WEB_GID:-1000} - volumes: - - *pi-web-dev-data-volume + volumes: *pi-web-dev-volumes sessiond: build: *pi-web-dev-build diff --git a/docker/internal/dev/sync-node-modules b/docker/internal/dev/sync-node-modules new file mode 100755 index 0000000..51baac5 --- /dev/null +++ b/docker/internal/dev/sync-node-modules @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH + +log() { + printf '%s\n' "$*" >&2 +} + +die() { + log "pi-web Docker dev dependencies: $*" + exit 1 +} + +workspace_dir=${PI_WEB_DEV_WORKSPACE_DIR:-/workspace} +seed_dir=${PI_WEB_DEV_DEPENDENCY_SEED_DIR:-/opt/pi-web-dev-dependencies} +target_dir=$workspace_dir/node_modules +generation_file=$seed_dir/generation +marker_file=$target_dir/.pi-web-dev-dependency-generation + +# A direct Compose invocation may skip the image rebuild. Fail closed rather +# than copying dependencies for different checkout manifests. +for manifest in package.json package-lock.json; do + source_manifest=$workspace_dir/$manifest + image_manifest=$seed_dir/$manifest + [ -f "$source_manifest" ] || die "checkout is missing $source_manifest" + [ -f "$image_manifest" ] || die "development image is missing $image_manifest" + if ! cmp -s "$source_manifest" "$image_manifest"; then + die "development image dependencies do not match the checkout; run ./docker/pi-web-docker --dev start or update to rebuild the image" + fi +done + +[ -d "$seed_dir/node_modules" ] || die "development image is missing the dependency seed at $seed_dir/node_modules" +[ -s "$generation_file" ] || die "development image is missing its dependency generation at $generation_file" +[ ! -L "$target_dir" ] || die "refusing to synchronize through the node_modules symlink at $target_dir" +mkdir -p "$target_dir" + +expected_generation=$(cat "$generation_file") +current_generation= +if [ -f "$marker_file" ]; then + current_generation=$(cat "$marker_file") +fi + +if [ "$current_generation" = "$expected_generation" ]; then + log "PI WEB Docker dev dependencies are current." + exit 0 +fi + +log "Synchronizing PI WEB Docker dev dependencies from the rebuilt image ..." +# Write the marker only after a complete copy so a failed init retries next time. +find "$target_dir" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +cp -a "$seed_dir/node_modules/." "$target_dir/" +printf '%s\n' "$expected_generation" >"$marker_file" +chmod 0666 "$marker_file" +log "PI WEB Docker dev dependencies synchronized." diff --git a/src/server/dockerControlAssets.test.ts b/src/server/dockerControlAssets.test.ts index f356a4d..4502c7a 100644 --- a/src/server/dockerControlAssets.test.ts +++ b/src/server/dockerControlAssets.test.ts @@ -39,20 +39,24 @@ describe("Docker command assets", () => { execUtf8("sh", ["-n", dockerEntrypoint], process.env), execUtf8("sh", ["-n", join(repoRoot, "docker", "install.sh")], process.env), execUtf8("sh", ["-n", join(repoRoot, "docker", "internal", "dev", "compose")], process.env), + execUtf8("bash", ["-n", join(repoRoot, "docker", "internal", "dev", "sync-node-modules")], process.env), execUtf8("sh", ["-n", join(repoRoot, "docker", "internal", "host-profile.sh")], process.env), ]); }); it("packages the canonical Docker command and internal support assets", async () => { - const [dockerfile, devDockerfile, runtimeCompose, devCompose, installer, devWrapper, dockerignore] = await Promise.all([ + const [dockerfile, devDockerfile, runtimeCompose, devCompose, installer, devWrapper, dependencySync, dockerignore] = await Promise.all([ readRepoFile("docker/Dockerfile"), readRepoFile("docker/Dockerfile.dev"), readRepoFile("docker/compose.yml"), readRepoFile("docker/compose.dev.yml"), readRepoFile("docker/install.sh"), readRepoFile("docker/internal/dev/compose"), + readRepoFile("docker/internal/dev/sync-node-modules"), readRepoFile("docker/.dockerignore"), ]); + const customImageHooksIndex = devDockerfile.indexOf("for script in /tmp/pi-web-custom-image.d/*.sh"); + const dependencyGenerationIndex = devDockerfile.indexOf("/opt/pi-web-dev-dependencies/generation"); expect(dockerfile).toContain("COPY pi-web-docker /usr/local/bin/pi-web-docker"); expect(dockerfile).toContain("COPY internal/bin/hostexec /usr/local/bin/hostexec"); @@ -62,6 +66,12 @@ describe("Docker command assets", () => { expect(dockerfile).not.toContain("@earendil-works/pi-coding-agent@"); expect(devDockerfile).toContain("COPY docker/pi-web-docker /usr/local/bin/pi-web-docker"); expect(devDockerfile).toContain("COPY docker/internal/bin/hostexec /usr/local/bin/hostexec"); + expect(devDockerfile).toContain("COPY --chmod=0755 docker/internal/dev/sync-node-modules /usr/local/sbin/pi-web-dev-sync-node-modules"); + expect(devDockerfile).toContain("/opt/pi-web-dev-dependencies/node_modules"); + // Hooks can mutate the dependency seed, so its cache generation must be finalized afterward. + expect(customImageHooksIndex).toBeGreaterThanOrEqual(0); + expect(dependencyGenerationIndex).toBeGreaterThan(customImageHooksIndex); + expect(dependencySync).toContain(".pi-web-dev-dependency-generation"); expect(dockerignore).toContain("!pi-web-docker"); expect(dockerignore).toContain("!internal/bin/hostexec"); expect(installer).toContain("write_asset pi-web-docker 0755"); @@ -83,6 +93,8 @@ describe("Docker command assets", () => { expect(devCompose).toContain("PI_WEB_DOCKER_DEV_REPO_ROOT: ${PI_WEB_DOCKER_DEV_REPO_ROOT:?set by docker/pi-web-docker --dev}"); expect(devCompose).toContain("PI_WEB_DOCKER_HELPER_IMAGE: ${PI_WEB_DEV_IMAGE:-pi-web:dev}"); expect(devCompose).toContain("COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-pi-web-dev}"); + expect(devCompose).toContain("/usr/local/sbin/pi-web-dev-sync-node-modules"); + expect(devCompose.match(/volumes: \*pi-web-dev-volumes/g)).toHaveLength(3); }); dockerCommandIt("fetches remote installer assets without clobbering the write target", async () => { diff --git a/src/server/dockerDevDependencySync.test.ts b/src/server/dockerDevDependencySync.test.ts new file mode 100644 index 0000000..d5ef6f2 --- /dev/null +++ b/src/server/dockerDevDependencySync.test.ts @@ -0,0 +1,102 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const syncScript = join(repoRoot, "docker", "internal", "dev", "sync-node-modules"); +const dockerSyncIt = it.skipIf(process.platform === "win32"); + +let tempDir = ""; + +interface SyncFixture { + workspaceDir: string; + seedDir: string; + targetDir: string; +} + +beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), "pi-web-docker-dependencies-test-")); +}); + +afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); +}); + +describe("Docker development dependency synchronization", () => { + dockerSyncIt("replaces a stale dependency tree once per image generation", async () => { + const fixture = await createSyncFixture(); + + const first = await runSync(fixture); + + expect(first.exitCode).toBe(0); + expect(first.stderr).toContain("Synchronizing PI WEB Docker dev dependencies"); + expect(await readFile(join(fixture.targetDir, "fresh", "version.txt"), "utf8")).toBe("0.80.6\n"); + expect(await readlink(join(fixture.targetDir, ".bin", "fresh"))).toBe("../fresh/version.txt"); + expect(await readFile(join(fixture.targetDir, ".pi-web-dev-dependency-generation"), "utf8")).toBe("image-generation-2\n"); + await expect(readFile(join(fixture.targetDir, "stale.txt"), "utf8")).rejects.toThrow(); + + await writeFile(join(fixture.targetDir, "keep-on-current-generation.txt"), "kept\n", "utf8"); + const second = await runSync(fixture); + + expect(second.exitCode).toBe(0); + expect(second.stderr).toContain("dependencies are current"); + expect(await readFile(join(fixture.targetDir, "keep-on-current-generation.txt"), "utf8")).toBe("kept\n"); + }); + + dockerSyncIt("fails without changing the volume when the image manifests are stale", async () => { + const fixture = await createSyncFixture(); + await writeFile(join(fixture.workspaceDir, "package-lock.json"), '{"lockfileVersion":3,"changed":true}\n', "utf8"); + + const result = await runSync(fixture); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("development image dependencies do not match the checkout"); + expect(await readFile(join(fixture.targetDir, "stale.txt"), "utf8")).toBe("stale\n"); + }); +}); + +async function createSyncFixture(): Promise { + const workspaceDir = join(tempDir, "workspace"); + const seedDir = join(tempDir, "seed"); + const targetDir = join(workspaceDir, "node_modules"); + const packageJson = '{"name":"dependency-sync-fixture","private":true}\n'; + const packageLock = '{"name":"dependency-sync-fixture","lockfileVersion":3}\n'; + + await Promise.all([ + mkdir(join(seedDir, "node_modules", "fresh"), { recursive: true }), + mkdir(join(seedDir, "node_modules", ".bin"), { recursive: true }), + mkdir(targetDir, { recursive: true }), + ]); + await Promise.all([ + writeFile(join(workspaceDir, "package.json"), packageJson, "utf8"), + writeFile(join(workspaceDir, "package-lock.json"), packageLock, "utf8"), + writeFile(join(seedDir, "package.json"), packageJson, "utf8"), + writeFile(join(seedDir, "package-lock.json"), packageLock, "utf8"), + writeFile(join(seedDir, "generation"), "image-generation-2\n", "utf8"), + writeFile(join(seedDir, "node_modules", "fresh", "version.txt"), "0.80.6\n", "utf8"), + writeFile(join(targetDir, "stale.txt"), "stale\n", "utf8"), + writeFile(join(targetDir, ".pi-web-dev-dependency-generation"), "image-generation-1\n", "utf8"), + ]); + await symlink("../fresh/version.txt", join(seedDir, "node_modules", ".bin", "fresh")); + + return { workspaceDir, seedDir, targetDir }; +} + +function runSync(fixture: SyncFixture): Promise<{ stdout: string; stderr: string; exitCode: number }> { + return new Promise((resolvePromise) => { + execFile("bash", [syncScript], { + encoding: "utf8", + env: { + ...process.env, + PI_WEB_DEV_WORKSPACE_DIR: fixture.workspaceDir, + PI_WEB_DEV_DEPENDENCY_SEED_DIR: fixture.seedDir, + }, + }, (error, stdout, stderr) => { + const exitCode = typeof error === "object" && error !== null && "code" in error && typeof error.code === "number" ? error.code : 0; + resolvePromise({ stdout, stderr, exitCode }); + }); + }); +}