feat(cli): replace the xgenext2fs spawn with @deroll/genext2fs

.changeset/witty-melons-repeat.md

@@ -0,0 +1,14 @@
+---
+"@cartesi/cli": minor
+---
+
+Replace the `xgenext2fs` subprocess with the `@deroll/genext2fs` bindings
+
+ext2 drives are now built through [`@deroll/genext2fs`](https://deroll.dev/genext2fs), an N-API
+addon, so building a drive no longer shells out to `xgenext2fs` and no longer falls back to
+running it inside the SDK Docker image. Docker is still required to build the root drive from a
+Dockerfile, and for squashfs drives (`mksquashfs`).
+
+Drive contents are unchanged: the bindings are given the same block size, faketime and
+readjustment settings the command line used, so a drive built before and after this change is
+byte identical.

CLAUDE.md

@@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts  # Run a single test
 bun run build --filter @cartesi/devnet
 ```
 
-The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` is left external — it is a native addon that resolves its platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries.
+The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` and `@deroll/genext2fs` are left external — they are native addons that resolve their platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries.
 
 ## Architecture
 
@@ -55,7 +55,7 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation
 -   **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`.
 -   **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives.
 -   **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.).
--   **`exec/`** — Machine and filesystem tooling. `cartesi-machine` and `cartesi-machine-stored-hash` are native N-API bindings (`@cartesi/machine`); `genext2fs`, `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image.
+-   **`exec/`** — Machine and filesystem tooling. `cartesi-machine`, `cartesi-machine-stored-hash` and `genext2fs` are native N-API bindings (`@cartesi/machine`, `@deroll/genext2fs`); `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image.
 -   **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives), mirroring what the `cartesi-machine` CLI does with its command line.
 -   **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release.
 -   **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions.

apps/cli/build.ts

@@ -1,6 +1,6 @@
-// the emulator binding resolves its platform binary at runtime, so it can never
-// be bundled: it is left as an import, resolved from node_modules
-const external = ["@cartesi/machine"];
+// native addons resolve their platform binary at runtime, so they can never be
+// bundled: they are left as imports, resolved from node_modules
+const external = ["@cartesi/machine", "@deroll/genext2fs"];
 
 // build for npm package
 await Bun.build({
@@ -14,9 +14,9 @@ await Bun.build({
 });
 
 // NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*)
-// are gone. A single file executable has no node_modules, and the emulator
-// binding resolves its platform specific .node at runtime, so it cannot be
-// embedded — not even for the host platform. The npm package is the only
-// distribution now, and the homebrew formula has to install it from there.
+// are gone. A single file executable has no node_modules, and the emulator and
+// ext2 bindings resolve their platform specific .node at runtime, so they
+// cannot be embedded — not even for the host platform. The npm package is the
+// only distribution now, and the homebrew formula has to install it from there.
 
 export {};

apps/cli/package.json

@@ -17,6 +17,7 @@
     "dependencies": {
         "@cartesi/machine": "^1.0.0-alpha.0",
         "@commander-js/extra-typings": "^14.0.0",
+        "@deroll/genext2fs": "^0.2.0-alpha.0",
         "@inquirer/confirm": "^6.0.6",
         "@inquirer/core": "^11.1.3",
         "@inquirer/input": "^5.0.6",

apps/cli/src/builder/directory.ts

@@ -27,7 +27,6 @@ export const build = async (
                     input: name,
                     output: filename,
                     cwd: destination,
-                    image: sdkImage,
                     reporter,
                 });
                 break;

apps/cli/src/builder/docker.ts

@@ -162,7 +162,6 @@ export const build = async (
                     input: tar,
                     output: filename,
                     cwd: destination,
-                    image: sdkImage,
                     reporter,
                 });
                 break;

apps/cli/src/builder/empty.ts

@@ -6,7 +6,6 @@ import { genext2fs } from "../exec/index.js";
 export const build = async (
     name: string,
     drive: EmptyDriveConfig,
-    sdkImage: string,
     destination: string,
 ): Promise<void> => {
     const filename = `${name}.${drive.format}`;
@@ -16,7 +15,6 @@ export const build = async (
                 output: filename,
                 size: drive.size,
                 cwd: destination,
-                image: sdkImage,
             });
             break;
         }

apps/cli/src/builder/tar.ts

@@ -24,7 +24,6 @@ export const build = async (
                 input: tar,
                 output: filename,
                 cwd: destination,
-                image: sdkImage,
                 reporter,
             });
             break;

apps/cli/src/commands/build.ts

@@ -63,7 +63,7 @@ const buildDriveTask = (
                 break;
             }
             case "empty": {
-                await buildEmpty(name, drive, sdk, destination);
+                await buildEmpty(name, drive, destination);
                 break;
             }
             case "tar": {

apps/cli/src/exec/genext2fs.ts

@@ -1,90 +1,97 @@
+import {
+    createImage,
+    type Genext2fsResult,
+    tarToExt2,
+    version as vendoredVersion,
+} from "@deroll/genext2fs";
+import path from "node:path";
 import { parse, Range, type SemVer } from "semver";
-import { type DockerFallbackOptions, execaDockerFallback } from "./util.js";
+import type { Reporter } from "./util.js";
 
 const BLOCK_SIZE = 4096; // fixed at 4k
 
 export const requiredVersion: Range = new Range("^1.5.6");
 
-const baseArgs = (options: { extraBlocks: number }) => [
-    "--block-size",
-    BLOCK_SIZE.toString(),
-    "--faketime",
-    "--readjustment",
-    `+${options.extraBlocks}`,
-];
+type BaseOptions = {
+    /** directory the input and output filenames are relative to */
+    cwd?: string;
+    reporter?: Reporter;
+};
+
+const resolve = (cwd: string | undefined, filename: string): string =>
+    cwd ? path.resolve(cwd, filename) : path.resolve(filename);
+
+/**
+ * Forwards the diagnostics xgenext2fs produced to the reporter, one line at a
+ * time, the same way the spawned process' stderr used to be piped.
+ */
+const report = (result: Genext2fsResult, reporter?: Reporter): void => {
+    if (!reporter) {
+        return;
+    }
+    for (const line of result.stderr.split("\n")) {
+        if (line.trim()) {
+            reporter(line.trimEnd());
+        }
+    }
+};
 
-export const empty = (
+export const empty = async (
     options: {
-        cwd?: string;
         size: number;
         output: string;
-    } & DockerFallbackOptions,
-) => {
-    const { size, output, reporter } = options;
+    } & BaseOptions,
+): Promise<Genext2fsResult> => {
+    const { cwd, size, output, reporter } = options;
     const blocks = Math.ceil(size / BLOCK_SIZE); // size in blocks
-    return execaDockerFallback(
-        "xgenext2fs",
-        [
-            "--block-size",
-            BLOCK_SIZE.toString(),
-            "--faketime",
-            "--size-in-blocks",
-            blocks.toString(),
-            output,
-        ],
-        { ...options, reporter },
-    );
+    const result = await createImage(resolve(cwd, output), {
+        blockSize: BLOCK_SIZE,
+        faketime: true,
+        sizeInBlocks: blocks,
+    });
+    report(result, reporter);
+    return result;
 };
 
-export const fromDirectory = (
+export const fromDirectory = async (
     options: {
-        cwd?: string;
         extraSize: number;
         input: string;
         output: string;
-    } & DockerFallbackOptions,
-) => {
-    const { cwd, extraSize, image, input, output, reporter } = options;
+    } & BaseOptions,
+): Promise<Genext2fsResult> => {
+    const { cwd, extraSize, input, output, reporter } = options;
     const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE);
-    return execaDockerFallback(
-        "xgenext2fs",
-        [...baseArgs({ extraBlocks }), "--root", input, output],
-        { cwd, image, reporter },
-    );
+    const result = await createImage(resolve(cwd, output), {
+        blockSize: BLOCK_SIZE,
+        faketime: true,
+        readjustment: `+${extraBlocks}`,
+        layers: [{ type: "directory", path: resolve(cwd, input) }],
+    });
+    report(result, reporter);
+    return result;
 };
 
-export const fromTar = (
+export const fromTar = async (
     options: {
-        cwd?: string;
         extraSize: number;
         input: string;
         output: string;
-    } & DockerFallbackOptions,
-) => {
-    const { cwd, extraSize, image, input, output, reporter } = options;
+    } & BaseOptions,
+): Promise<Genext2fsResult> => {
+    const { cwd, extraSize, input, output, reporter } = options;
     const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE);
-    return execaDockerFallback(
-        "xgenext2fs",
-        [...baseArgs({ extraBlocks }), "--tarball", input, output],
-        { cwd, image, reporter },
-    );
+    const result = await tarToExt2(resolve(cwd, input), resolve(cwd, output), {
+        blockSize: BLOCK_SIZE,
+        faketime: true,
+        readjustment: `+${extraBlocks}`,
+    });
+    report(result, reporter);
+    return result;
 };
 
-export const version = async (
-    options?: DockerFallbackOptions,
-): Promise<SemVer | null> => {
-    const { stdout } = await execaDockerFallback(
-        "xgenext2fs",
-        ["--version"],
-        options || {},
-    );
-    if (typeof stdout === "string") {
-        const regex =
-            /(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?/;
-        const m = stdout.match(regex);
-        if (m?.[0]) {
-            return parse(m[0]);
-        }
-    }
-    return null;
-};
+/**
+ * Version of the xgenext2fs the bindings were built against. It is fixed at
+ * build time, so this is a plain lookup and not a subprocess call anymore.
+ */
+export const version = (): SemVer | null => parse(vendoredVersion);

apps/cli/tests/integration/builder/empty.test.ts

@@ -1,27 +1,11 @@
-import {
-    afterEach,
-    beforeAll,
-    beforeEach,
-    describe,
-    expect,
-    it,
-} from "bun:test";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import fs from "fs-extra";
 import path from "node:path";
 import { build } from "../../../src/builder/empty.js";
 import type { EmptyDriveConfig } from "../../../src/config.js";
-import { setupIntegrationTests, TEST_SDK } from "../config.js";
 import { cleanupTempDir, createTempDir } from "./tmpdirTest.js";
 
-beforeAll(
-    async () => {
-        await setupIntegrationTests();
-    },
-    { timeout: 60000 },
-);
-
 describe("when building with the empty builder", () => {
-    const image = TEST_SDK;
     let destination: string;
 
     beforeEach(async () => {
@@ -38,7 +22,7 @@ describe("when building with the empty builder", () => {
             format: "ext2",
             size: 0,
         };
-        await expect(build("root", drive, image, destination)).rejects.toThrow(
+        await expect(build("root", drive, destination)).rejects.toThrow(
             "too few blocks",
         );
     });
@@ -50,7 +34,7 @@ describe("when building with the empty builder", () => {
             format: "ext2",
             size: 1024 * 1024 * 1, // 1Mb
         };
-        await build("root", drive, image, destination);
+        await build("root", drive, destination);
 
         const filename = path.join(destination, driveName);
         expect(fs.existsSync(filename)).toBeTruthy();
@@ -66,7 +50,7 @@ describe("when building with the empty builder", () => {
             format: "raw",
             size: 1024 * 1024 * 1, // 1Mb
         };
-        await build("root", drive, image, destination);
+        await build("root", drive, destination);
 
         const filename = path.join(destination, driveName);
         expect(fs.existsSync(filename)).toBeTruthy();