diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index ddc5bd6c..154626ad 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -369,24 +369,43 @@ export async function materializePlaygroundStagedFiles(server: PlaygroundCliServ for (const mount of mounts) { if (mount.type !== "file") continue - const handle = await open(mount.source, "r") + // The target can be a writable NodeFS mount of mount.source. The first + // truncating write would otherwise shorten the file being streamed. + const snapshotDirectory = await mkdtemp(join(tmpdir(), "wp-codebox-staged-file-")) + const snapshot = join(snapshotDirectory, basename(mount.source) || "staged-file") try { - const buffer = Buffer.allocUnsafe(STAGED_FILE_CHUNK_SIZE) - let position = 0 - let append = false - do { - const { bytesRead } = await handle.read(buffer, 0, buffer.length, position) - const response = await server.playground.run({ code: stagedFileWritePhp(mount.target, buffer.subarray(0, bytesRead).toString("base64"), append) }) - const result = JSON.parse(response.text || "{}") as { schema?: string; written?: number } - if (result.schema !== "wp-codebox/staged-file-write/v1" || result.written !== bytesRead) { - throw new Error(`Could not materialize staged file at ${mount.target}`) + await cp(mount.source, snapshot) + const handle = await open(snapshot, "r") + try { + const buffer = Buffer.allocUnsafe(STAGED_FILE_CHUNK_SIZE) + const hash = createHash("sha256") + let position = 0 + let append = false + do { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position) + const contents = buffer.subarray(0, bytesRead) + hash.update(contents) + const response = await server.playground.run({ code: stagedFileWritePhp(mount.target, contents.toString("base64"), append) }) + const result = JSON.parse(response.text || "{}") as { schema?: string; written?: number } + if (result.schema !== "wp-codebox/staged-file-write/v1" || result.written !== bytesRead) { + throw new Error(`Could not materialize staged file at ${mount.target}`) + } + position += bytesRead + append = true + if (bytesRead === 0 || bytesRead < buffer.length) break + } while (true) + + const response = await server.playground.run({ code: stagedFileVerificationPhp(mount.target) }) + const result = JSON.parse(response.text || "{}") as { schema?: string; bytes?: number; sha256?: string } + const expectedSha256 = hash.digest("hex") + if (result.schema !== "wp-codebox/staged-file-verification/v1" || result.bytes !== position || result.sha256 !== expectedSha256) { + throw new Error(`Staged file verification failed at ${mount.target}: expected ${position} bytes sha256 ${expectedSha256}, received ${typeof result.bytes === "number" ? result.bytes : "unknown"} bytes sha256 ${typeof result.sha256 === "string" ? result.sha256 : "unknown"}`) } - position += bytesRead - append = true - if (bytesRead === 0 || bytesRead < buffer.length) break - } while (true) + } finally { + await handle.close() + } } finally { - await handle.close() + await rm(snapshotDirectory, { recursive: true, force: true }) } materialized++ } @@ -410,6 +429,25 @@ echo json_encode(array('schema' => 'wp-codebox/staged-file-write/v1', 'written' ` } +function stagedFileVerificationPhp(target: string): string { + const payload = JSON.stringify(JSON.stringify({ target })) + return ` 'wp-codebox/staged-file-verification/v1', 'bytes' => $bytes, 'sha256' => $sha256), JSON_UNESCAPED_SLASHES); +` +} + function nestedMountPaths(mounts: MountSpec[], mountIndex: number, parentTarget: string): string[] { const normalizedParent = parentTarget.replace(/\/+$/, "") return mounts diff --git a/tests/staged-file-materialization.test.ts b/tests/staged-file-materialization.test.ts new file mode 100644 index 00000000..de42e57c --- /dev/null +++ b/tests/staged-file-materialization.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" + +import { materializePlaygroundStagedFiles } from "../packages/runtime-playground/src/mount-materialization.js" + +const root = await mkdtemp(join(tmpdir(), "wp-codebox-staged-file-materialization-")) + +try { + const text = Buffer.from(`{"fixture":"${"text-".repeat(60_000)}end"}\n`) + const binary = Buffer.allocUnsafe(256 * 1024 + 137) + for (let index = 0; index < binary.length; index++) binary[index] = (index * 31 + 255) % 256 + + for (const [name, contents] of [["artifact.json", text], ["artifact.bin", binary]] as const) { + const source = join(root, name) + const target = `/workspace/${name}` + await writeFile(source, contents) + const sandboxFiles = new Map() + + const materialized = await materializePlaygroundStagedFiles({ + playground: { + async run({ code }) { + const payload = JSON.parse(JSON.parse(code.match(/\$payload = json_decode\((.*), true\);/)?.[1] ?? "\"{}\"")) as { target: string; contentsBase64?: string; append?: boolean } + if (code.includes("wp-codebox/staged-file-write/v1")) { + const chunk = Buffer.from(payload.contentsBase64 ?? "", "base64") + const existing = payload.append ? sandboxFiles.get(payload.target) ?? Buffer.alloc(0) : Buffer.alloc(0) + const next = Buffer.concat([existing, chunk]) + sandboxFiles.set(payload.target, next) + // This simulates the mounted target overwriting its host source. + await writeFile(source, next) + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-write/v1", written: chunk.length }) } + } + const actual = sandboxFiles.get(payload.target) ?? Buffer.alloc(0) + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-verification/v1", bytes: actual.length, sha256: createHash("sha256").update(actual).digest("hex") }) } + }, + }, + } as never, [{ type: "file", source, target, mode: "readwrite" }]) + + assert.equal(materialized, 1) + assert.deepEqual(sandboxFiles.get(target), contents, `${name} is written across all chunks`) + assert.deepEqual(await readFile(source), contents, `${name} remains byte-identical after mounted writes`) + } + + const truncatedSource = join(root, "truncated.json") + await writeFile(truncatedSource, text) + await assert.rejects(materializePlaygroundStagedFiles({ + playground: { + async run({ code }) { + const payload = JSON.parse(JSON.parse(code.match(/\$payload = json_decode\((.*), true\);/)?.[1] ?? "\"{}\"")) as { contentsBase64?: string } + if (code.includes("wp-codebox/staged-file-write/v1")) { + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-write/v1", written: Buffer.from(payload.contentsBase64 ?? "", "base64").length }) } + } + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-verification/v1", bytes: 262144, sha256: "0".repeat(64) }) } + }, + }, + } as never, [{ type: "file", source: truncatedSource, target: "/workspace/truncated.json", mode: "readwrite" }]), /expected \d+ bytes sha256 [a-f0-9]{64}, received 262144 bytes sha256 0{64}/) +} finally { + await rm(root, { recursive: true, force: true }) +} + +console.log("staged file materialization preserves large text and binary files")