diff --git a/.changeset/witty-melons-repeat.md b/.changeset/witty-melons-repeat.md new file mode 100644 index 00000000..ee292cb3 --- /dev/null +++ b/.changeset/witty-melons-repeat.md @@ -0,0 +1,14 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `xgenext2fs` subprocess with the `@deroll/genext2fs` bindings + +ext2 drives are now built through [`@deroll/genext2fs`](https://deroll.dev/genext2fs), an N-API +addon, so building a drive no longer shells out to `xgenext2fs` and no longer falls back to +running it inside the SDK Docker image. Docker is still required to build the root drive from a +Dockerfile, and for squashfs drives (`mksquashfs`). + +Drive contents are unchanged: the bindings are given the same block size, faketime and +readjustment settings the command line used, so a drive built before and after this change is +byte identical. diff --git a/CLAUDE.md b/CLAUDE.md index cff0f50b..3f3ab074 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` is left external — it is a native addon that resolves its platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` and `@deroll/genext2fs` are left external — they are native addons that resolve their platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,7 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Machine and filesystem tooling. `cartesi-machine` and `cartesi-machine-stored-hash` are native N-API bindings (`@cartesi/machine`); `genext2fs`, `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine`, `cartesi-machine-stored-hash` and `genext2fs` are native N-API bindings (`@cartesi/machine`, `@deroll/genext2fs`); `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. - **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives), mirroring what the `cartesi-machine` CLI does with its command line. - **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 2bd8b21c..37c87eb6 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,6 +1,6 @@ -// the emulator binding resolves its platform binary at runtime, so it can never -// be bundled: it is left as an import, resolved from node_modules -const external = ["@cartesi/machine"]; +// native addons resolve their platform binary at runtime, so they can never be +// bundled: they are left as imports, resolved from node_modules +const external = ["@cartesi/machine", "@deroll/genext2fs"]; // build for npm package await Bun.build({ @@ -14,9 +14,9 @@ await Bun.build({ }); // NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) -// are gone. A single file executable has no node_modules, and the emulator -// binding resolves its platform specific .node at runtime, so it cannot be -// embedded — not even for the host platform. The npm package is the only -// distribution now, and the homebrew formula has to install it from there. +// are gone. A single file executable has no node_modules, and the emulator and +// ext2 bindings resolve their platform specific .node at runtime, so they +// cannot be embedded — not even for the host platform. The npm package is the +// only distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 1c202603..55ef31b3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -17,6 +17,7 @@ "dependencies": { "@cartesi/machine": "^1.0.0-alpha.0", "@commander-js/extra-typings": "^14.0.0", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", diff --git a/apps/cli/src/builder/directory.ts b/apps/cli/src/builder/directory.ts index 55eb5398..8204714f 100644 --- a/apps/cli/src/builder/directory.ts +++ b/apps/cli/src/builder/directory.ts @@ -27,7 +27,6 @@ export const build = async ( input: name, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index 908b3acd..2f891b6b 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -162,7 +162,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/empty.ts b/apps/cli/src/builder/empty.ts index 9ddc0475..b52cb4f0 100644 --- a/apps/cli/src/builder/empty.ts +++ b/apps/cli/src/builder/empty.ts @@ -6,7 +6,6 @@ import { genext2fs } from "../exec/index.js"; export const build = async ( name: string, drive: EmptyDriveConfig, - sdkImage: string, destination: string, ): Promise => { const filename = `${name}.${drive.format}`; @@ -16,7 +15,6 @@ export const build = async ( output: filename, size: drive.size, cwd: destination, - image: sdkImage, }); break; } diff --git a/apps/cli/src/builder/tar.ts b/apps/cli/src/builder/tar.ts index 34633ecc..fe34c8f7 100644 --- a/apps/cli/src/builder/tar.ts +++ b/apps/cli/src/builder/tar.ts @@ -24,7 +24,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index 00ebcdcb..b0e3c16d 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -63,7 +63,7 @@ const buildDriveTask = ( break; } case "empty": { - await buildEmpty(name, drive, sdk, destination); + await buildEmpty(name, drive, destination); break; } case "tar": { diff --git a/apps/cli/src/exec/genext2fs.ts b/apps/cli/src/exec/genext2fs.ts index 99ba584e..ff2caa8b 100644 --- a/apps/cli/src/exec/genext2fs.ts +++ b/apps/cli/src/exec/genext2fs.ts @@ -1,90 +1,97 @@ +import { + createImage, + type Genext2fsResult, + tarToExt2, + version as vendoredVersion, +} from "@deroll/genext2fs"; +import path from "node:path"; import { parse, Range, type SemVer } from "semver"; -import { type DockerFallbackOptions, execaDockerFallback } from "./util.js"; +import type { Reporter } from "./util.js"; const BLOCK_SIZE = 4096; // fixed at 4k export const requiredVersion: Range = new Range("^1.5.6"); -const baseArgs = (options: { extraBlocks: number }) => [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--readjustment", - `+${options.extraBlocks}`, -]; +type BaseOptions = { + /** directory the input and output filenames are relative to */ + cwd?: string; + reporter?: Reporter; +}; + +const resolve = (cwd: string | undefined, filename: string): string => + cwd ? path.resolve(cwd, filename) : path.resolve(filename); + +/** + * Forwards the diagnostics xgenext2fs produced to the reporter, one line at a + * time, the same way the spawned process' stderr used to be piped. + */ +const report = (result: Genext2fsResult, reporter?: Reporter): void => { + if (!reporter) { + return; + } + for (const line of result.stderr.split("\n")) { + if (line.trim()) { + reporter(line.trimEnd()); + } + } +}; -export const empty = ( +export const empty = async ( options: { - cwd?: string; size: number; output: string; - } & DockerFallbackOptions, -) => { - const { size, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, size, output, reporter } = options; const blocks = Math.ceil(size / BLOCK_SIZE); // size in blocks - return execaDockerFallback( - "xgenext2fs", - [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--size-in-blocks", - blocks.toString(), - output, - ], - { ...options, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + sizeInBlocks: blocks, + }); + report(result, reporter); + return result; }; -export const fromDirectory = ( +export const fromDirectory = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--root", input, output], - { cwd, image, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + layers: [{ type: "directory", path: resolve(cwd, input) }], + }); + report(result, reporter); + return result; }; -export const fromTar = ( +export const fromTar = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--tarball", input, output], - { cwd, image, reporter }, - ); + const result = await tarToExt2(resolve(cwd, input), resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + }); + report(result, reporter); + return result; }; -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { stdout } = await execaDockerFallback( - "xgenext2fs", - ["--version"], - options || {}, - ); - if (typeof stdout === "string") { - const regex = - /(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?/; - const m = stdout.match(regex); - if (m?.[0]) { - return parse(m[0]); - } - } - return null; -}; +/** + * Version of the xgenext2fs the bindings were built against. It is fixed at + * build time, so this is a plain lookup and not a subprocess call anymore. + */ +export const version = (): SemVer | null => parse(vendoredVersion); diff --git a/apps/cli/tests/integration/builder/empty.test.ts b/apps/cli/tests/integration/builder/empty.test.ts index 88186ca4..30a0e7a8 100644 --- a/apps/cli/tests/integration/builder/empty.test.ts +++ b/apps/cli/tests/integration/builder/empty.test.ts @@ -1,27 +1,11 @@ -import { - afterEach, - beforeAll, - beforeEach, - describe, - expect, - it, -} from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import fs from "fs-extra"; import path from "node:path"; import { build } from "../../../src/builder/empty.js"; import type { EmptyDriveConfig } from "../../../src/config.js"; -import { setupIntegrationTests, TEST_SDK } from "../config.js"; import { cleanupTempDir, createTempDir } from "./tmpdirTest.js"; -beforeAll( - async () => { - await setupIntegrationTests(); - }, - { timeout: 60000 }, -); - describe("when building with the empty builder", () => { - const image = TEST_SDK; let destination: string; beforeEach(async () => { @@ -38,7 +22,7 @@ describe("when building with the empty builder", () => { format: "ext2", size: 0, }; - await expect(build("root", drive, image, destination)).rejects.toThrow( + await expect(build("root", drive, destination)).rejects.toThrow( "too few blocks", ); }); @@ -50,7 +34,7 @@ describe("when building with the empty builder", () => { format: "ext2", size: 1024 * 1024 * 1, // 1Mb }; - await build("root", drive, image, destination); + await build("root", drive, destination); const filename = path.join(destination, driveName); expect(fs.existsSync(filename)).toBeTruthy(); @@ -66,7 +50,7 @@ describe("when building with the empty builder", () => { format: "raw", size: 1024 * 1024 * 1, // 1Mb }; - await build("root", drive, image, destination); + await build("root", drive, destination); const filename = path.join(destination, driveName); expect(fs.existsSync(filename)).toBeTruthy(); diff --git a/apps/cli/tests/integration/exec/genext2fs.test.ts b/apps/cli/tests/integration/exec/genext2fs.test.ts deleted file mode 100644 index b5b4cbdb..00000000 --- a/apps/cli/tests/integration/exec/genext2fs.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { beforeAll, describe, expect, it } from "bun:test"; -import { type SemVer, satisfies } from "semver"; -import { genext2fs } from "../../../src/exec/index.js"; -import { setupIntegrationTests, TEST_SDK } from "../config.js"; - -beforeAll( - async () => { - await setupIntegrationTests(); - }, - { timeout: 60000 }, -); - -describe("genext2fs", () => { - it("should report version", async () => { - const version = await genext2fs.version({ - forceDocker: true, - image: TEST_SDK, - }); - - expect(version).toBeDefined(); - expect( - satisfies((version as SemVer).format(), genext2fs.requiredVersion), - ).toBeTruthy(); - }); -}); diff --git a/apps/cli/tests/unit/exec/genext2fs.test.ts b/apps/cli/tests/unit/exec/genext2fs.test.ts new file mode 100644 index 00000000..86f0575c --- /dev/null +++ b/apps/cli/tests/unit/exec/genext2fs.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import fs from "fs-extra"; +import path from "node:path"; +import { type SemVer, satisfies } from "semver"; +import { genext2fs } from "../../../src/exec/index.js"; + +describe("genext2fs", () => { + let destination: string; + + beforeEach(async () => { + destination = await fs.mkdtemp(path.join(__dirname, "genext2fs-")); + }); + + afterEach(async () => { + await fs.remove(destination); + }); + + it("should report the version of the bundled xgenext2fs", () => { + const version = genext2fs.version(); + + expect(version).toBeDefined(); + expect( + satisfies((version as SemVer).format(), genext2fs.requiredVersion), + ).toBeTruthy(); + }); + + it("should create an empty drive of the requested size", async () => { + await genext2fs.empty({ + cwd: destination, + output: "data.ext2", + size: 1024 * 1024, + }); + + const stat = await fs.stat(path.join(destination, "data.ext2")); + expect(stat.isFile()).toBeTruthy(); + expect(stat.size).toEqual(1024 * 1024); + }); + + it("should fail to create an empty drive with no blocks", async () => { + await expect( + genext2fs.empty({ + cwd: destination, + output: "data.ext2", + size: 0, + }), + ).rejects.toThrow("too few blocks"); + }); + + it("should create a drive from a directory", async () => { + const input = path.join(destination, "input"); + await fs.outputFile(path.join(input, "hello.txt"), "hello"); + + await genext2fs.fromDirectory({ + cwd: destination, + extraSize: 1024 * 1024, + input: "input", + output: "data.ext2", + }); + + const stat = await fs.stat(path.join(destination, "data.ext2")); + expect(stat.isFile()).toBeTruthy(); + expect(stat.size).toBeGreaterThan(0); + }); + + it("should produce reproducible drives", async () => { + const input = path.join(destination, "input"); + await fs.outputFile(path.join(input, "hello.txt"), "hello"); + + const outputs = ["one.ext2", "two.ext2"]; + for (const output of outputs) { + await genext2fs.fromDirectory({ + cwd: destination, + extraSize: 1024 * 1024, + input: "input", + output, + }); + } + + const [one, two] = await Promise.all( + outputs.map((output) => + fs.readFile(path.join(destination, output)), + ), + ); + expect(one.equals(two)).toBeTruthy(); + }); +}); diff --git a/bun.lock b/bun.lock index 624b37a4..0bc5030f 100644 --- a/bun.lock +++ b/bun.lock @@ -20,6 +20,7 @@ "dependencies": { "@cartesi/machine": "^1.0.0-alpha.0", "@commander-js/extra-typings": "^14.0.0", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", @@ -196,6 +197,8 @@ "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@deroll/genext2fs": ["@deroll/genext2fs@0.2.0-alpha.0", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-cfFKQngg2QU28yZWHHeBI3DcSrc+eCqiUEvDEioIdRlkPuol+QO+X1EM3wJrxfQNtj36rTyCvQK38KblDmDETA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],