diff --git a/package.json b/package.json index 7b3835fa..c4a8a1e7 100644 --- a/package.json +++ b/package.json @@ -276,6 +276,7 @@ "test:composer-installed-versions-loader-order": "tsx scripts/composer-installed-versions-loader-order-smoke.ts", "test:recipe-extra-plugin-composer-autoloaders": "tsx tests/recipe-extra-plugin-composer-autoloaders.test.ts", "test:recipe-extra-plugin-local-zip": "tsx tests/recipe-extra-plugin-local-zip.test.ts", + "test:zip-source-policy": "tsx tests/zip-source-policy.test.ts", "test:runtime-preset-registry": "tsx tests/runtime-preset-registry.test.ts", "test:generic-ability-runtime-run": "tsx tests/generic-ability-runtime-run.test.ts", "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts", diff --git a/packages/cli/src/recipe-sources.ts b/packages/cli/src/recipe-sources.ts index a87224c3..50397679 100644 --- a/packages/cli/src/recipe-sources.ts +++ b/packages/cli/src/recipe-sources.ts @@ -7,10 +7,10 @@ import type { MountSpec, WorkspaceRecipe, WorkspaceRecipeDependencyOverlay, Work import { executeManagedHostCommand, resolvePluginEntrypointContract } from "@automattic/wp-codebox-core" import { collectPreparedSourceCleanupPaths, DEFAULT_PREPARED_SOURCE_EXCLUDE_NAMES, localPreparedSourceProvenance, prepareLocalSourceStageSync, SANDBOX_WORKSPACE_ROOT, type PreparedSourceProvenance } from "@automattic/wp-codebox-core/internals" import { registerRuntimeOverlayDescriptor, runtimeOverlayDescriptor } from "./runtime-overlay-registry.js" -import { evaluateSourcePolicy, evaluateZipSourcePolicy, sourcePolicySnapshot, type SourcePolicyIssue } from "./source-policy.js" +import { evaluateSourcePolicy, evaluateZipSourcePolicy, sourcePolicySnapshot, type ArchiveSourceClass, type SourcePolicyIssue } from "./source-policy.js" import { prepareLocalZipSource, prepareZipSource } from "./zip-source.js" -export { ALLOW_NETWORK_DOWNLOADS_ENV, ALLOWED_DOWNLOAD_HOSTS_ENV, allowedDownloadHosts, isSha256, maxDownloadBytes, maxExtractedBytes, maxExtractedFiles, MAX_DOWNLOAD_BYTES_ENV, MAX_EXTRACTED_BYTES_ENV, MAX_EXTRACTED_FILES_ENV, REQUIRE_SOURCE_SHA256_ENV, sourceSha256Required } from "./source-policy.js" +export { ALLOW_NETWORK_DOWNLOADS_ENV, ALLOWED_DOWNLOAD_HOSTS_ENV, allowedDownloadHosts, archiveSourceClass, isSha256, maxCompressionRatio, maxDownloadBytes, maxExtractedBytes, maxExtractedFileBytes, maxExtractedFiles, maxExtractedFilesFor, MAX_COMPRESSION_RATIO_ENV, MAX_DOWNLOAD_BYTES_ENV, MAX_EXTRACTED_BYTES_ENV, MAX_EXTRACTED_FILES_ENV, MAX_EXTRACTED_FILE_BYTES_ENV, REQUIRE_SOURCE_SHA256_ENV, sourceSha256Required, TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES_ENV } from "./source-policy.js" const PHP_AI_CLIENT_RUNTIME_OVERLAY_TARGET = "/wordpress/wp-includes/php-ai-client" const PHP_SCOPER_DOWNLOAD_ATTEMPTS = 3 @@ -46,6 +46,9 @@ export interface RecipeSourceProvenance { maxDownloadBytes: number maxExtractedBytes: number maxExtractedFiles: number + maxExtractedFileBytes: number + maxCompressionRatio: number + archiveClass: ArchiveSourceClass sha256Required: boolean } localPathCategory?: "recipe-relative" | "temporary-download" | "temporary-composer-autoload" @@ -124,6 +127,7 @@ export interface ParsedRecipeSource { host: string expectedSha256?: string wporgSlug?: string + archiveClass?: ArchiveSourceClass } const PHP_SCOPER_VERSION = "0.18.17" @@ -1442,7 +1446,7 @@ async function prepareRecipeSource(sourceRef: string, recipeDirectory: string, s if (policyIssue) { throw new Error(policyIssue.message) } - const preparedZip = await prepareLocalZipSource(localPath, slug, source.expectedSha256) + const preparedZip = await prepareLocalZipSource(localPath, slug, source.expectedSha256, source.archiveClass) return { source: await extractedPluginSourceDirectory(preparedZip.extractDirectory, slug), cleanupPaths: [preparedZip.root], @@ -1472,7 +1476,7 @@ async function prepareRecipeSource(sourceRef: string, recipeDirectory: string, s provenance: { ...recipeSourceProvenance(source, recipeDirectory), digest: { sha256: preparedZip.digest, ...(source.expectedSha256 ? { expected: source.expectedSha256, verified: true } : {}) }, - policy: sourcePolicySnapshot(source.host), + policy: sourcePolicySnapshot(source.host, source.archiveClass), localPathCategory: "temporary-download", }, } @@ -1597,7 +1601,8 @@ export function recipeSource(sourceRef: string, expectedSha256?: string): Parsed try { url = new URL(sourceRef) } catch { - return { type: "local", resolvedUrl: sourceRef, host: "", ...(expectedSha256 ? { expectedSha256: expectedSha256.toLowerCase() } : {}) } + const normalizedSha256 = expectedSha256?.toLowerCase() + return { type: "local", resolvedUrl: sourceRef, host: "", ...(normalizedSha256 ? { expectedSha256: normalizedSha256, archiveClass: "trusted" } : {}) } } if (url.protocol !== "https:") { @@ -1611,7 +1616,7 @@ export function recipeSource(sourceRef: string, expectedSha256?: string): Parsed if (url.hostname === "downloads.wordpress.org" && url.pathname.startsWith("/plugin/")) { const filename = basename(url.pathname) const match = filename.match(/^([A-Za-z0-9_-]+)\./) - return { type: "wporg_plugin_zip", resolvedUrl: url.toString(), host: url.hostname, ...(expectedSha256 ? { expectedSha256: expectedSha256.toLowerCase() } : {}), ...(match ? { wporgSlug: match[1] } : {}) } + return { type: "wporg_plugin_zip", resolvedUrl: url.toString(), host: url.hostname, archiveClass: "trusted", ...(expectedSha256 ? { expectedSha256: expectedSha256.toLowerCase() } : {}), ...(match ? { wporgSlug: match[1] } : {}) } } return { type: "https_zip", resolvedUrl: url.toString(), host: url.hostname, ...(expectedSha256 ? { expectedSha256: expectedSha256.toLowerCase() } : {}) } @@ -1661,7 +1666,7 @@ export function recipeSourceProvenance(source: ParsedRecipeSource, recipeDirecto original: source.resolvedUrl, resolvedUrl: source.resolvedUrl, ...(source.expectedSha256 ? { digest: { sha256: source.expectedSha256, expected: source.expectedSha256, verified: false } } : {}), - policy: sourcePolicySnapshot(source.host), + policy: sourcePolicySnapshot(source.host, source.archiveClass), } } diff --git a/packages/cli/src/source-policy.ts b/packages/cli/src/source-policy.ts index 2b5bd6c2..2d04647c 100644 --- a/packages/cli/src/source-policy.ts +++ b/packages/cli/src/source-policy.ts @@ -1,8 +1,11 @@ export interface ExternalSourcePolicyInput { type: string host: string + archiveClass?: ArchiveSourceClass } +export type ArchiveSourceClass = "standard" | "trusted" + export interface SourcePolicyIssue { code: string message: string @@ -10,9 +13,12 @@ export interface SourcePolicyIssue { export interface SourcePolicySnapshot { host: string + archiveClass: ArchiveSourceClass maxDownloadBytes: number maxExtractedBytes: number maxExtractedFiles: number + maxExtractedFileBytes: number + maxCompressionRatio: number sha256Required: boolean } @@ -22,11 +28,17 @@ export const REQUIRE_SOURCE_SHA256_ENV = "WP_CODEBOX_REQUIRE_SOURCE_SHA256" export const MAX_DOWNLOAD_BYTES_ENV = "WP_CODEBOX_MAX_DOWNLOAD_BYTES" export const MAX_EXTRACTED_BYTES_ENV = "WP_CODEBOX_MAX_EXTRACTED_BYTES" export const MAX_EXTRACTED_FILES_ENV = "WP_CODEBOX_MAX_EXTRACTED_FILES" +export const TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES_ENV = "WP_CODEBOX_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES" +export const MAX_EXTRACTED_FILE_BYTES_ENV = "WP_CODEBOX_MAX_EXTRACTED_FILE_BYTES" +export const MAX_COMPRESSION_RATIO_ENV = "WP_CODEBOX_MAX_COMPRESSION_RATIO" const DEFAULT_ALLOWED_DOWNLOAD_HOSTS = ["downloads.wordpress.org"] const DEFAULT_MAX_DOWNLOAD_BYTES = 25 * 1024 * 1024 const DEFAULT_MAX_EXTRACTED_BYTES = 100 * 1024 * 1024 const DEFAULT_MAX_EXTRACTED_FILES = 5000 +const DEFAULT_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES = 10_000 +const DEFAULT_MAX_EXTRACTED_FILE_BYTES = 25 * 1024 * 1024 +const DEFAULT_MAX_COMPRESSION_RATIO = 100 export function isSha256(value: string): boolean { return /^[a-f0-9]{64}$/i.test(value) @@ -119,12 +131,34 @@ export function maxExtractedFiles(): number { return envPositiveInteger(MAX_EXTRACTED_FILES_ENV, DEFAULT_MAX_EXTRACTED_FILES) } -export function sourcePolicySnapshot(host: string): SourcePolicySnapshot { +export function archiveSourceClass(source: ExternalSourcePolicyInput): ArchiveSourceClass { + return source.archiveClass === "trusted" ? "trusted" : "standard" +} + +export function maxExtractedFilesFor(source: ExternalSourcePolicyInput): number { + return archiveSourceClass(source) === "trusted" + ? envPositiveInteger(TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES_ENV, DEFAULT_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES) + : maxExtractedFiles() +} + +export function maxExtractedFileBytes(): number { + return envPositiveInteger(MAX_EXTRACTED_FILE_BYTES_ENV, DEFAULT_MAX_EXTRACTED_FILE_BYTES) +} + +export function maxCompressionRatio(): number { + return envPositiveInteger(MAX_COMPRESSION_RATIO_ENV, DEFAULT_MAX_COMPRESSION_RATIO) +} + +export function sourcePolicySnapshot(host: string, archiveClass: ArchiveSourceClass = "standard"): SourcePolicySnapshot { + const source = { type: "archive", host, archiveClass } return { host, + archiveClass, maxDownloadBytes: maxDownloadBytes(), maxExtractedBytes: maxExtractedBytes(), - maxExtractedFiles: maxExtractedFiles(), + maxExtractedFiles: maxExtractedFilesFor(source), + maxExtractedFileBytes: maxExtractedFileBytes(), + maxCompressionRatio: maxCompressionRatio(), sha256Required: sourceSha256Required(), } } diff --git a/packages/cli/src/zip-source.ts b/packages/cli/src/zip-source.ts index 50b7009f..1f7ac2e2 100644 --- a/packages/cli/src/zip-source.ts +++ b/packages/cli/src/zip-source.ts @@ -3,13 +3,14 @@ import { lstat, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "n import { tmpdir } from "node:os" import { dirname, join } from "node:path" import { executeManagedHostCommand } from "@automattic/wp-codebox-core" -import { allowedDownloadHosts, maxDownloadBytes, maxExtractedBytes, maxExtractedFiles } from "./source-policy.js" +import { allowedDownloadHosts, maxCompressionRatio, maxDownloadBytes, maxExtractedBytes, maxExtractedFileBytes, maxExtractedFilesFor, type ArchiveSourceClass } from "./source-policy.js" export interface ZipSourceReference { type: string resolvedUrl: string host: string expectedSha256?: string + archiveClass?: ArchiveSourceClass } export interface PreparedZipSource { @@ -27,14 +28,14 @@ export async function prepareZipSource(sourc const extractDirectory = join(root, "extracted") await mkdir(extractDirectory, { recursive: true }) const digest = await downloadZipSource(source, zipPath, redirectSource) - await assertSafeZipEntries(zipPath) + await assertSafeZipEntries(zipPath, source) await executeManagedHostCommand({ command: "unzip", args: ["-q", zipPath, "-d", extractDirectory], cwd: root, allowedCwdRoots: [root], label: "extract recipe source zip" }) - await assertExtractedSourceBounds(extractDirectory) + await assertExtractedSourceBounds(extractDirectory, source) return { root, zipPath, extractDirectory, digest } } -export async function prepareLocalZipSource(sourcePath: string, slug: string, expectedSha256?: string): Promise { +export async function prepareLocalZipSource(sourcePath: string, slug: string, expectedSha256?: string, archiveClass: ArchiveSourceClass = "standard"): Promise { const root = await mkdtemp(join(tmpdir(), `wp-codebox-source-${slug}-`)) const zipPath = join(root, "source.zip") const extractDirectory = join(root, "extracted") @@ -50,9 +51,10 @@ export async function prepareLocalZipSource(sourcePath: string, slug: string, ex } await writeFile(zipPath, buffer) await mkdir(extractDirectory, { recursive: true }) - await assertSafeZipEntries(zipPath) + const archiveSource = { type: "local", resolvedUrl: sourcePath, host: "", archiveClass } + await assertSafeZipEntries(zipPath, archiveSource) await executeManagedHostCommand({ command: "unzip", args: ["-q", zipPath, "-d", extractDirectory], cwd: root, allowedCwdRoots: [root], label: "extract recipe source zip" }) - await assertExtractedSourceBounds(extractDirectory) + await assertExtractedSourceBounds(extractDirectory, archiveSource) return { root, zipPath, extractDirectory, digest } } catch (error) { await rm(root, { recursive: true, force: true }) @@ -91,26 +93,85 @@ async function downloadZipSource(source: TSo return digest } -async function assertSafeZipEntries(zipPath: string): Promise { - const root = dirname(zipPath) - const { stdout } = await executeManagedHostCommand({ command: "unzip", args: ["-Z1", zipPath], cwd: root, allowedCwdRoots: [root], label: "list recipe source zip" }) - const entries = stdout.split(/\r?\n/).filter(Boolean) - if (entries.length > maxExtractedFiles()) { - throw new Error(`Recipe source zip contains too many entries: ${entries.length}`) +async function assertSafeZipEntries(zipPath: string, source: ZipSourceReference): Promise { + const entries = zipEntries(await readFile(zipPath)) + const maxFiles = maxExtractedFilesFor(source) + if (entries.length > maxFiles) { + throw new Error(`Recipe source zip contains too many entries: ${entries.length}; limit ${maxFiles}; archive class ${source.archiveClass ?? "standard"}`) } for (const entry of entries) { - const normalized = entry.replace(/\\/g, "/") + const normalized = entry.name.replace(/\\/g, "/") if (normalized.startsWith("/") || normalized.split("/").includes("..")) { - throw new Error(`Recipe source zip contains an unsafe path: ${entry}`) + throw new Error(`Recipe source zip contains an unsafe path: ${entry.name}`) } } + + const expandedBytes = entries.reduce((total, entry) => total + entry.uncompressedBytes, 0) + if (expandedBytes > maxExtractedBytes()) { + throw new Error(`Recipe source extraction exceeds ${maxExtractedBytes()} bytes: ${expandedBytes}`) + } + + for (const { compressedBytes, uncompressedBytes } of entries) { + if (uncompressedBytes > maxExtractedFileBytes()) { + throw new Error(`Recipe source zip entry exceeds ${maxExtractedFileBytes()} bytes: ${uncompressedBytes}`) + } + if (uncompressedBytes > 0 && (compressedBytes === 0 || uncompressedBytes / compressedBytes > maxCompressionRatio())) { + throw new Error(`Recipe source zip entry exceeds ${maxCompressionRatio()}:1 compression ratio`) + } + } +} + +function zipEntries(data: Buffer): Array<{ name: string; compressedBytes: number; uncompressedBytes: number }> { + const minimumEndOfCentralDirectory = 22 + const endOfCentralDirectory = findEndOfCentralDirectory(data) + if (endOfCentralDirectory < 0 || data.length < minimumEndOfCentralDirectory) { + throw new Error("Recipe source zip has no valid central directory") + } + + const disk = data.readUInt16LE(endOfCentralDirectory + 4) + const centralDirectoryDisk = data.readUInt16LE(endOfCentralDirectory + 6) + const entriesOnDisk = data.readUInt16LE(endOfCentralDirectory + 8) + const entryCount = data.readUInt16LE(endOfCentralDirectory + 10) + const centralDirectoryBytes = data.readUInt32LE(endOfCentralDirectory + 12) + let offset = data.readUInt32LE(endOfCentralDirectory + 16) + if (disk !== 0 || centralDirectoryDisk !== 0 || entriesOnDisk !== entryCount || entryCount === 0xffff || centralDirectoryBytes === 0xffffffff || offset === 0xffffffff) { + throw new Error("Recipe source zip uses an unsupported central directory") + } + + const end = offset + centralDirectoryBytes + if (!Number.isSafeInteger(end) || end > endOfCentralDirectory) throw new Error("Recipe source zip has an invalid central directory range") + + const entries: Array<{ name: string; compressedBytes: number; uncompressedBytes: number }> = [] + for (let index = 0; index < entryCount; index += 1) { + if (offset + 46 > end || data.readUInt32LE(offset) !== 0x02014b50) throw new Error("Recipe source zip has an invalid central directory entry") + const compressedBytes = data.readUInt32LE(offset + 20) + const uncompressedBytes = data.readUInt32LE(offset + 24) + const nameBytes = data.readUInt16LE(offset + 28) + const extraBytes = data.readUInt16LE(offset + 30) + const commentBytes = data.readUInt16LE(offset + 32) + if (compressedBytes === 0xffffffff || uncompressedBytes === 0xffffffff) throw new Error("Recipe source zip uses an unsupported ZIP64 entry") + entries.push({ name: data.toString("utf8", offset + 46, offset + 46 + nameBytes), compressedBytes, uncompressedBytes }) + offset += 46 + nameBytes + extraBytes + commentBytes + } + + if (offset !== end) throw new Error("Recipe source zip has an invalid central directory size") + return entries +} + +function findEndOfCentralDirectory(data: Buffer): number { + const earliest = Math.max(0, data.length - 0xffff - 22) + for (let offset = data.length - 22; offset >= earliest; offset -= 1) { + if (data.readUInt32LE(offset) === 0x06054b50 && offset + 22 + data.readUInt16LE(offset + 20) === data.length) return offset + } + return -1 } -async function assertExtractedSourceBounds(directory: string): Promise { +async function assertExtractedSourceBounds(directory: string, source: ZipSourceReference): Promise { const totals = await directoryTotals(directory) - if (totals.files > maxExtractedFiles()) { - throw new Error(`Recipe source extraction contains too many files: ${totals.files}`) + const maxFiles = maxExtractedFilesFor(source) + if (totals.files > maxFiles) { + throw new Error(`Recipe source extraction contains too many files: ${totals.files}; limit ${maxFiles}; archive class ${source.archiveClass ?? "standard"}`) } if (totals.bytes > maxExtractedBytes()) { throw new Error(`Recipe source extraction exceeds ${maxExtractedBytes()} bytes: ${totals.bytes}`) diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index ddc5bd6c..154626ad 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -369,24 +369,43 @@ export async function materializePlaygroundStagedFiles(server: PlaygroundCliServ for (const mount of mounts) { if (mount.type !== "file") continue - const handle = await open(mount.source, "r") + // The target can be a writable NodeFS mount of mount.source. The first + // truncating write would otherwise shorten the file being streamed. + const snapshotDirectory = await mkdtemp(join(tmpdir(), "wp-codebox-staged-file-")) + const snapshot = join(snapshotDirectory, basename(mount.source) || "staged-file") try { - const buffer = Buffer.allocUnsafe(STAGED_FILE_CHUNK_SIZE) - let position = 0 - let append = false - do { - const { bytesRead } = await handle.read(buffer, 0, buffer.length, position) - const response = await server.playground.run({ code: stagedFileWritePhp(mount.target, buffer.subarray(0, bytesRead).toString("base64"), append) }) - const result = JSON.parse(response.text || "{}") as { schema?: string; written?: number } - if (result.schema !== "wp-codebox/staged-file-write/v1" || result.written !== bytesRead) { - throw new Error(`Could not materialize staged file at ${mount.target}`) + await cp(mount.source, snapshot) + const handle = await open(snapshot, "r") + try { + const buffer = Buffer.allocUnsafe(STAGED_FILE_CHUNK_SIZE) + const hash = createHash("sha256") + let position = 0 + let append = false + do { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position) + const contents = buffer.subarray(0, bytesRead) + hash.update(contents) + const response = await server.playground.run({ code: stagedFileWritePhp(mount.target, contents.toString("base64"), append) }) + const result = JSON.parse(response.text || "{}") as { schema?: string; written?: number } + if (result.schema !== "wp-codebox/staged-file-write/v1" || result.written !== bytesRead) { + throw new Error(`Could not materialize staged file at ${mount.target}`) + } + position += bytesRead + append = true + if (bytesRead === 0 || bytesRead < buffer.length) break + } while (true) + + const response = await server.playground.run({ code: stagedFileVerificationPhp(mount.target) }) + const result = JSON.parse(response.text || "{}") as { schema?: string; bytes?: number; sha256?: string } + const expectedSha256 = hash.digest("hex") + if (result.schema !== "wp-codebox/staged-file-verification/v1" || result.bytes !== position || result.sha256 !== expectedSha256) { + throw new Error(`Staged file verification failed at ${mount.target}: expected ${position} bytes sha256 ${expectedSha256}, received ${typeof result.bytes === "number" ? result.bytes : "unknown"} bytes sha256 ${typeof result.sha256 === "string" ? result.sha256 : "unknown"}`) } - position += bytesRead - append = true - if (bytesRead === 0 || bytesRead < buffer.length) break - } while (true) + } finally { + await handle.close() + } } finally { - await handle.close() + await rm(snapshotDirectory, { recursive: true, force: true }) } materialized++ } @@ -410,6 +429,25 @@ echo json_encode(array('schema' => 'wp-codebox/staged-file-write/v1', 'written' ` } +function stagedFileVerificationPhp(target: string): string { + const payload = JSON.stringify(JSON.stringify({ target })) + return ` 'wp-codebox/staged-file-verification/v1', 'bytes' => $bytes, 'sha256' => $sha256), JSON_UNESCAPED_SLASHES); +` +} + function nestedMountPaths(mounts: MountSpec[], mountIndex: number, parentTarget: string): string[] { const normalizedParent = parentTarget.replace(/\/+$/, "") return mounts diff --git a/scripts/source-policy-smoke.ts b/scripts/source-policy-smoke.ts index 8347397f..258970ca 100644 --- a/scripts/source-policy-smoke.ts +++ b/scripts/source-policy-smoke.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict" import { ALLOWED_DOWNLOAD_HOSTS_ENV, ALLOW_NETWORK_DOWNLOADS_ENV, evaluateSourcePolicy, REQUIRE_SOURCE_SHA256_ENV, sourcePolicySnapshot } from "../packages/cli/src/source-policy.js" +import { recipeSource } from "../packages/cli/src/recipe-sources.js" const originalEnv = { [ALLOW_NETWORK_DOWNLOADS_ENV]: process.env[ALLOW_NETWORK_DOWNLOADS_ENV], @@ -25,6 +26,9 @@ try { assert.deepEqual(evaluateSourcePolicy({ type: "https_zip", host: "example.com" }, "a".repeat(64)), []) assert.equal(sourcePolicySnapshot("example.com").host, "example.com") assert.equal(sourcePolicySnapshot("example.com").sha256Required, true) + assert.equal(sourcePolicySnapshot("example.com", "trusted").archiveClass, "trusted") + assert.equal(sourcePolicySnapshot("example.com", "trusted").maxExtractedFiles, 10_000) + assert.equal(recipeSource("package.zip", "a".repeat(64)).archiveClass, "trusted") } finally { for (const [name, value] of Object.entries(originalEnv)) { if (value === undefined) { diff --git a/tests/staged-file-materialization.test.ts b/tests/staged-file-materialization.test.ts new file mode 100644 index 00000000..de42e57c --- /dev/null +++ b/tests/staged-file-materialization.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" + +import { materializePlaygroundStagedFiles } from "../packages/runtime-playground/src/mount-materialization.js" + +const root = await mkdtemp(join(tmpdir(), "wp-codebox-staged-file-materialization-")) + +try { + const text = Buffer.from(`{"fixture":"${"text-".repeat(60_000)}end"}\n`) + const binary = Buffer.allocUnsafe(256 * 1024 + 137) + for (let index = 0; index < binary.length; index++) binary[index] = (index * 31 + 255) % 256 + + for (const [name, contents] of [["artifact.json", text], ["artifact.bin", binary]] as const) { + const source = join(root, name) + const target = `/workspace/${name}` + await writeFile(source, contents) + const sandboxFiles = new Map() + + const materialized = await materializePlaygroundStagedFiles({ + playground: { + async run({ code }) { + const payload = JSON.parse(JSON.parse(code.match(/\$payload = json_decode\((.*), true\);/)?.[1] ?? "\"{}\"")) as { target: string; contentsBase64?: string; append?: boolean } + if (code.includes("wp-codebox/staged-file-write/v1")) { + const chunk = Buffer.from(payload.contentsBase64 ?? "", "base64") + const existing = payload.append ? sandboxFiles.get(payload.target) ?? Buffer.alloc(0) : Buffer.alloc(0) + const next = Buffer.concat([existing, chunk]) + sandboxFiles.set(payload.target, next) + // This simulates the mounted target overwriting its host source. + await writeFile(source, next) + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-write/v1", written: chunk.length }) } + } + const actual = sandboxFiles.get(payload.target) ?? Buffer.alloc(0) + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-verification/v1", bytes: actual.length, sha256: createHash("sha256").update(actual).digest("hex") }) } + }, + }, + } as never, [{ type: "file", source, target, mode: "readwrite" }]) + + assert.equal(materialized, 1) + assert.deepEqual(sandboxFiles.get(target), contents, `${name} is written across all chunks`) + assert.deepEqual(await readFile(source), contents, `${name} remains byte-identical after mounted writes`) + } + + const truncatedSource = join(root, "truncated.json") + await writeFile(truncatedSource, text) + await assert.rejects(materializePlaygroundStagedFiles({ + playground: { + async run({ code }) { + const payload = JSON.parse(JSON.parse(code.match(/\$payload = json_decode\((.*), true\);/)?.[1] ?? "\"{}\"")) as { contentsBase64?: string } + if (code.includes("wp-codebox/staged-file-write/v1")) { + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-write/v1", written: Buffer.from(payload.contentsBase64 ?? "", "base64").length }) } + } + return { text: JSON.stringify({ schema: "wp-codebox/staged-file-verification/v1", bytes: 262144, sha256: "0".repeat(64) }) } + }, + }, + } as never, [{ type: "file", source: truncatedSource, target: "/workspace/truncated.json", mode: "readwrite" }]), /expected \d+ bytes sha256 [a-f0-9]{64}, received 262144 bytes sha256 0{64}/) +} finally { + await rm(root, { recursive: true, force: true }) +} + +console.log("staged file materialization preserves large text and binary files") diff --git a/tests/zip-source-policy.test.ts b/tests/zip-source-policy.test.ts new file mode 100644 index 00000000..73a846e1 --- /dev/null +++ b/tests/zip-source-policy.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { mkdir, rm, writeFile } from "node:fs/promises" +import { join } from "node:path" +import { promisify } from "node:util" + +import { prepareLocalZipSource } from "../packages/cli/src/zip-source.js" +import { withTempDir } from "../scripts/test-kit.js" + +const execFileAsync = promisify(execFile) + +async function archive(directory: string, name: string, entries: string[]): Promise { + await execFileAsync("zip", ["-q", name, ...entries], { cwd: directory }) + return join(directory, name) +} + +await withTempDir("wp-codebox-zip-source-policy-", async (directory) => { + const entries = Array.from({ length: 5002 }, (_, index) => `entry-${String(index).padStart(5, "0")}.txt`) + await mkdir(join(directory, "package")) + await Promise.all(entries.map((entry) => writeFile(join(directory, "package", entry), "x"))) + + const validArchive = await archive(join(directory, "package"), "../valid.zip", entries.slice(0, 5001)) + const prepared = await prepareLocalZipSource(validArchive, "valid", undefined, "trusted") + await rm(prepared.root, { recursive: true, force: true }) + + const overLimitArchive = await archive(join(directory, "package"), "../over-limit.zip", entries) + const previousLimit = process.env.WP_CODEBOX_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES + process.env.WP_CODEBOX_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES = "5001" + try { + await assert.rejects(() => prepareLocalZipSource(overLimitArchive, "over-limit", undefined, "trusted"), /contains too many entries: 5002; limit 5001; archive class trusted/) + } finally { + if (previousLimit === undefined) delete process.env.WP_CODEBOX_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES + else process.env.WP_CODEBOX_TRUSTED_ARCHIVE_MAX_EXTRACTED_FILES = previousLimit + } +}) + +console.log("zip source policy bounds ok")