From 590525b9ff4f8a7b7ee3c3b0612430d67eb4d468 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:38:41 +0000 Subject: [PATCH] feat(cli): replace the xgenext2fs spawn with @deroll/genext2fs Build ext2 drives through the @deroll/genext2fs N-API bindings, instead of spawning xgenext2fs (falling back to running it inside the SDK docker image). The bindings are given the same block size, faketime and readjustment settings the command line used, so drives stay byte identical. Since building a drive no longer needs docker, the version test moves from the integration suite to the unit one, where it also covers building images and their reproducibility. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UTEd5g3mF849BATTstssR3 --- .changeset/witty-melons-repeat.md | 14 ++ CLAUDE.md | 4 +- apps/cli/build.ts | 14 +- apps/cli/package.json | 1 + apps/cli/src/builder/directory.ts | 1 - apps/cli/src/builder/docker.ts | 1 - apps/cli/src/builder/empty.ts | 2 - apps/cli/src/builder/tar.ts | 1 - apps/cli/src/commands/build.ts | 2 +- apps/cli/src/exec/genext2fs.ts | 133 +++++++++--------- .../tests/integration/builder/empty.test.ts | 24 +--- .../tests/integration/exec/genext2fs.test.ts | 25 ---- apps/cli/tests/unit/exec/genext2fs.test.ts | 86 +++++++++++ bun.lock | 3 + 14 files changed, 188 insertions(+), 123 deletions(-) create mode 100644 .changeset/witty-melons-repeat.md delete mode 100644 apps/cli/tests/integration/exec/genext2fs.test.ts create mode 100644 apps/cli/tests/unit/exec/genext2fs.test.ts diff --git a/.changeset/witty-melons-repeat.md b/.changeset/witty-melons-repeat.md new file mode 100644 index 00000000..ee292cb3 --- /dev/null +++ b/.changeset/witty-melons-repeat.md @@ -0,0 +1,14 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `xgenext2fs` subprocess with the `@deroll/genext2fs` bindings + +ext2 drives are now built through [`@deroll/genext2fs`](https://deroll.dev/genext2fs), an N-API +addon, so building a drive no longer shells out to `xgenext2fs` and no longer falls back to +running it inside the SDK Docker image. Docker is still required to build the root drive from a +Dockerfile, and for squashfs drives (`mksquashfs`). + +Drive contents are unchanged: the bindings are given the same block size, faketime and +readjustment settings the command line used, so a drive built before and after this change is +byte identical. diff --git a/CLAUDE.md b/CLAUDE.md index cff0f50b..3f3ab074 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` is left external — it is a native addon that resolves its platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` and `@deroll/genext2fs` are left external — they are native addons that resolve their platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,7 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Machine and filesystem tooling. `cartesi-machine` and `cartesi-machine-stored-hash` are native N-API bindings (`@cartesi/machine`); `genext2fs`, `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine`, `cartesi-machine-stored-hash` and `genext2fs` are native N-API bindings (`@cartesi/machine`, `@deroll/genext2fs`); `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. - **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives), mirroring what the `cartesi-machine` CLI does with its command line. - **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 2bd8b21c..37c87eb6 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,6 +1,6 @@ -// the emulator binding resolves its platform binary at runtime, so it can never -// be bundled: it is left as an import, resolved from node_modules -const external = ["@cartesi/machine"]; +// native addons resolve their platform binary at runtime, so they can never be +// bundled: they are left as imports, resolved from node_modules +const external = ["@cartesi/machine", "@deroll/genext2fs"]; // build for npm package await Bun.build({ @@ -14,9 +14,9 @@ await Bun.build({ }); // NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) -// are gone. A single file executable has no node_modules, and the emulator -// binding resolves its platform specific .node at runtime, so it cannot be -// embedded — not even for the host platform. The npm package is the only -// distribution now, and the homebrew formula has to install it from there. +// are gone. A single file executable has no node_modules, and the emulator and +// ext2 bindings resolve their platform specific .node at runtime, so they +// cannot be embedded — not even for the host platform. The npm package is the +// only distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 1c202603..55ef31b3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -17,6 +17,7 @@ "dependencies": { "@cartesi/machine": "^1.0.0-alpha.0", "@commander-js/extra-typings": "^14.0.0", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", diff --git a/apps/cli/src/builder/directory.ts b/apps/cli/src/builder/directory.ts index 55eb5398..8204714f 100644 --- a/apps/cli/src/builder/directory.ts +++ b/apps/cli/src/builder/directory.ts @@ -27,7 +27,6 @@ export const build = async ( input: name, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index 908b3acd..2f891b6b 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -162,7 +162,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/empty.ts b/apps/cli/src/builder/empty.ts index 9ddc0475..b52cb4f0 100644 --- a/apps/cli/src/builder/empty.ts +++ b/apps/cli/src/builder/empty.ts @@ -6,7 +6,6 @@ import { genext2fs } from "../exec/index.js"; export const build = async ( name: string, drive: EmptyDriveConfig, - sdkImage: string, destination: string, ): Promise => { const filename = `${name}.${drive.format}`; @@ -16,7 +15,6 @@ export const build = async ( output: filename, size: drive.size, cwd: destination, - image: sdkImage, }); break; } diff --git a/apps/cli/src/builder/tar.ts b/apps/cli/src/builder/tar.ts index 34633ecc..fe34c8f7 100644 --- a/apps/cli/src/builder/tar.ts +++ b/apps/cli/src/builder/tar.ts @@ -24,7 +24,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index 00ebcdcb..b0e3c16d 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -63,7 +63,7 @@ const buildDriveTask = ( break; } case "empty": { - await buildEmpty(name, drive, sdk, destination); + await buildEmpty(name, drive, destination); break; } case "tar": { diff --git a/apps/cli/src/exec/genext2fs.ts b/apps/cli/src/exec/genext2fs.ts index 99ba584e..ff2caa8b 100644 --- a/apps/cli/src/exec/genext2fs.ts +++ b/apps/cli/src/exec/genext2fs.ts @@ -1,90 +1,97 @@ +import { + createImage, + type Genext2fsResult, + tarToExt2, + version as vendoredVersion, +} from "@deroll/genext2fs"; +import path from "node:path"; import { parse, Range, type SemVer } from "semver"; -import { type DockerFallbackOptions, execaDockerFallback } from "./util.js"; +import type { Reporter } from "./util.js"; const BLOCK_SIZE = 4096; // fixed at 4k export const requiredVersion: Range = new Range("^1.5.6"); -const baseArgs = (options: { extraBlocks: number }) => [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--readjustment", - `+${options.extraBlocks}`, -]; +type BaseOptions = { + /** directory the input and output filenames are relative to */ + cwd?: string; + reporter?: Reporter; +}; + +const resolve = (cwd: string | undefined, filename: string): string => + cwd ? path.resolve(cwd, filename) : path.resolve(filename); + +/** + * Forwards the diagnostics xgenext2fs produced to the reporter, one line at a + * time, the same way the spawned process' stderr used to be piped. + */ +const report = (result: Genext2fsResult, reporter?: Reporter): void => { + if (!reporter) { + return; + } + for (const line of result.stderr.split("\n")) { + if (line.trim()) { + reporter(line.trimEnd()); + } + } +}; -export const empty = ( +export const empty = async ( options: { - cwd?: string; size: number; output: string; - } & DockerFallbackOptions, -) => { - const { size, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, size, output, reporter } = options; const blocks = Math.ceil(size / BLOCK_SIZE); // size in blocks - return execaDockerFallback( - "xgenext2fs", - [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--size-in-blocks", - blocks.toString(), - output, - ], - { ...options, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + sizeInBlocks: blocks, + }); + report(result, reporter); + return result; }; -export const fromDirectory = ( +export const fromDirectory = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--root", input, output], - { cwd, image, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + layers: [{ type: "directory", path: resolve(cwd, input) }], + }); + report(result, reporter); + return result; }; -export const fromTar = ( +export const fromTar = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--tarball", input, output], - { cwd, image, reporter }, - ); + const result = await tarToExt2(resolve(cwd, input), resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + }); + report(result, reporter); + return result; }; -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { stdout } = await execaDockerFallback( - "xgenext2fs", - ["--version"], - options || {}, - ); - if (typeof stdout === "string") { - const regex = - /(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?/; - const m = stdout.match(regex); - if (m?.[0]) { - return parse(m[0]); - } - } - return null; -}; +/** + * Version of the xgenext2fs the bindings were built against. It is fixed at + * build time, so this is a plain lookup and not a subprocess call anymore. + */ +export const version = (): SemVer | null => parse(vendoredVersion); diff --git a/apps/cli/tests/integration/builder/empty.test.ts b/apps/cli/tests/integration/builder/empty.test.ts index 88186ca4..30a0e7a8 100644 --- a/apps/cli/tests/integration/builder/empty.test.ts +++ b/apps/cli/tests/integration/builder/empty.test.ts @@ -1,27 +1,11 @@ -import { - afterEach, - beforeAll, - beforeEach, - describe, - expect, - it, -} from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import fs from "fs-extra"; import path from "node:path"; import { build } from "../../../src/builder/empty.js"; import type { EmptyDriveConfig } from "../../../src/config.js"; -import { setupIntegrationTests, TEST_SDK } from "../config.js"; import { cleanupTempDir, createTempDir } from "./tmpdirTest.js"; -beforeAll( - async () => { - await setupIntegrationTests(); - }, - { timeout: 60000 }, -); - describe("when building with the empty builder", () => { - const image = TEST_SDK; let destination: string; beforeEach(async () => { @@ -38,7 +22,7 @@ describe("when building with the empty builder", () => { format: "ext2", size: 0, }; - await expect(build("root", drive, image, destination)).rejects.toThrow( + await expect(build("root", drive, destination)).rejects.toThrow( "too few blocks", ); }); @@ -50,7 +34,7 @@ describe("when building with the empty builder", () => { format: "ext2", size: 1024 * 1024 * 1, // 1Mb }; - await build("root", drive, image, destination); + await build("root", drive, destination); const filename = path.join(destination, driveName); expect(fs.existsSync(filename)).toBeTruthy(); @@ -66,7 +50,7 @@ describe("when building with the empty builder", () => { format: "raw", size: 1024 * 1024 * 1, // 1Mb }; - await build("root", drive, image, destination); + await build("root", drive, destination); const filename = path.join(destination, driveName); expect(fs.existsSync(filename)).toBeTruthy(); diff --git a/apps/cli/tests/integration/exec/genext2fs.test.ts b/apps/cli/tests/integration/exec/genext2fs.test.ts deleted file mode 100644 index b5b4cbdb..00000000 --- a/apps/cli/tests/integration/exec/genext2fs.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import { type SemVer, satisfies } from "semver"; -import { genext2fs } from "../../../src/exec/index.js"; -import { setupIntegrationTests, TEST_SDK } from "../config.js"; - -beforeAll( - async () => { - await setupIntegrationTests(); - }, - { timeout: 60000 }, -); - -describe("genext2fs", () => { - it("should report version", async () => { - const version = await genext2fs.version({ - forceDocker: true, - image: TEST_SDK, - }); - - expect(version).toBeDefined(); - expect( - satisfies((version as SemVer).format(), genext2fs.requiredVersion), - ).toBeTruthy(); - }); -}); diff --git a/apps/cli/tests/unit/exec/genext2fs.test.ts b/apps/cli/tests/unit/exec/genext2fs.test.ts new file mode 100644 index 00000000..86f0575c --- /dev/null +++ b/apps/cli/tests/unit/exec/genext2fs.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import fs from "fs-extra"; +import path from "node:path"; +import { type SemVer, satisfies } from "semver"; +import { genext2fs } from "../../../src/exec/index.js"; + +describe("genext2fs", () => { + let destination: string; + + beforeEach(async () => { + destination = await fs.mkdtemp(path.join(__dirname, "genext2fs-")); + }); + + afterEach(async () => { + await fs.remove(destination); + }); + + it("should report the version of the bundled xgenext2fs", () => { + const version = genext2fs.version(); + + expect(version).toBeDefined(); + expect( + satisfies((version as SemVer).format(), genext2fs.requiredVersion), + ).toBeTruthy(); + }); + + it("should create an empty drive of the requested size", async () => { + await genext2fs.empty({ + cwd: destination, + output: "data.ext2", + size: 1024 * 1024, + }); + + const stat = await fs.stat(path.join(destination, "data.ext2")); + expect(stat.isFile()).toBeTruthy(); + expect(stat.size).toEqual(1024 * 1024); + }); + + it("should fail to create an empty drive with no blocks", async () => { + await expect( + genext2fs.empty({ + cwd: destination, + output: "data.ext2", + size: 0, + }), + ).rejects.toThrow("too few blocks"); + }); + + it("should create a drive from a directory", async () => { + const input = path.join(destination, "input"); + await fs.outputFile(path.join(input, "hello.txt"), "hello"); + + await genext2fs.fromDirectory({ + cwd: destination, + extraSize: 1024 * 1024, + input: "input", + output: "data.ext2", + }); + + const stat = await fs.stat(path.join(destination, "data.ext2")); + expect(stat.isFile()).toBeTruthy(); + expect(stat.size).toBeGreaterThan(0); + }); + + it("should produce reproducible drives", async () => { + const input = path.join(destination, "input"); + await fs.outputFile(path.join(input, "hello.txt"), "hello"); + + const outputs = ["one.ext2", "two.ext2"]; + for (const output of outputs) { + await genext2fs.fromDirectory({ + cwd: destination, + extraSize: 1024 * 1024, + input: "input", + output, + }); + } + + const [one, two] = await Promise.all( + outputs.map((output) => + fs.readFile(path.join(destination, output)), + ), + ); + expect(one.equals(two)).toBeTruthy(); + }); +}); diff --git a/bun.lock b/bun.lock index 624b37a4..0bc5030f 100644 --- a/bun.lock +++ b/bun.lock @@ -20,6 +20,7 @@ "dependencies": { "@cartesi/machine": "^1.0.0-alpha.0", "@commander-js/extra-typings": "^14.0.0", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", @@ -196,6 +197,8 @@ "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@deroll/genext2fs": ["@deroll/genext2fs@0.2.0-alpha.0", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-cfFKQngg2QU28yZWHHeBI3DcSrc+eCqiUEvDEioIdRlkPuol+QO+X1EM3wJrxfQNtj36rTyCvQK38KblDmDETA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],