New benchmark: a port of Richards to Kotlin/Wasm

.github/workflows/benchmarks.yml

@@ -75,7 +75,7 @@ jobs:
       run: |
         cargo run benchmark \
         --engine=artifacts/wasmtime-main/libengine.so \
-        --engine-flags "-Wgc -Wfunction-references" \
+        --engine-flags "-Wgc -Wfunction-references -Wexceptions" \
         --processes=4 \
         --iterations-per-process=1 \
         -- "${PARTITION_SUITE}"

benchmarks/all.suite

@@ -13,6 +13,7 @@ cm-online-stats/cm-online-stats.wasm
 hex-simd/benchmark.wasm
 # image-classification/image-classification-benchmark.wasm
 intgemm-simd/benchmark.wasm
+kotlin-richards/kotlin-richards.wasm
 libsodium/libsodium-aead_aegis128l.wasm
 libsodium/libsodium-aead_aegis256.wasm
 libsodium/libsodium-aead_aes256gcm.wasm

benchmarks/build.sh

@@ -31,7 +31,15 @@ print_header() {
 # directory and `--dereference` (i.e., follow) all symlinks provided.
 print_header "Create build context"
 TMP_TAR=$(mktemp /tmp/sightglass-benchmark-dir-XXXXXX.tar)
-(set -x; cd $BENCHMARK_DIR && tar --create --file $TMP_TAR --dereference --verbose .)
+# macOS's bsdtar bundles extended attributes (notably `com.apple.provenance`) that a Linux Docker
+# daemon rejects when unpacking the build context (`lsetxattr ... operation not supported`); exclude
+# them. GNU tar (used on CI) omits xattrs by default and lacks `--no-mac-metadata`, so only pass
+# these flags when the local `tar` is bsdtar.
+TAR_XATTR_FLAGS=()
+if tar --version 2>&1 | grep -qi bsdtar; then
+    TAR_XATTR_FLAGS=(--no-xattrs --no-mac-metadata)
+fi
+(set -x; cd $BENCHMARK_DIR && tar "${TAR_XATTR_FLAGS[@]}" --create --file $TMP_TAR --dereference --verbose .)
 
 # Build the benchmark image and extract the generated `benchmark.wasm` file from its container.
 print_header "Build benchmarks"

benchmarks/kotlin-richards/Dockerfile

@@ -0,0 +1,76 @@
+# Reproducible build of the kotlin-richards benchmark.
+#
+# Stage 1 compiles the Kotlin sources to an optimized WasmGC core module via the
+# Kotlin/Wasm `wasmWasi` target. Stage 2 turns that core module into a WASI
+# *component* with wasm-tools and the WASI preview1 command adapter.
+
+# ---------------------------------------------------------------------------
+# Stage 1: Kotlin -> optimized WasmGC core module.
+# ---------------------------------------------------------------------------
+FROM gradle:8.13-jdk21 AS kotlin-build
+WORKDIR /src
+
+# Warm the Kotlin/Native + Binaryen toolchain and dependency caches in a layer
+# that does not depend on our sources, so editing the Kotlin only re-runs the
+# (fast) compile below.
+COPY settings.gradle.kts gradle.properties build.gradle.kts ./
+RUN mkdir -p src/wasmWasiMain/kotlin \
+ && printf 'fun main() {}\n' > src/wasmWasiMain/kotlin/Stub.kt \
+ && gradle --no-daemon compileProductionExecutableKotlinWasmWasiOptimize \
+ && rm -rf src
+
+# Compile the real sources. Produces, with Binaryen already applied:
+#   build/compileSync/wasmWasi/main/productionExecutable/optimized/kotlin-richards.wasm
+COPY src ./src
+RUN gradle --no-daemon compileProductionExecutableKotlinWasmWasiOptimize
+
+# ---------------------------------------------------------------------------
+# Stage 2: core module -> WASI component.
+# ---------------------------------------------------------------------------
+FROM debian:bookworm-slim AS component-build
+
+# Pinned tool versions. The adapter targets WASI 0.2.6 (matches sightglass).
+ARG WASM_TOOLS_VERSION=1.247.0
+ARG ADAPTER_WASMTIME_VERSION=37.0.0
+WORKDIR /work
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends curl xz-utils ca-certificates; \
+    rm -rf /var/lib/apt/lists/*; \
+    case "$(uname -m)" in \
+      x86_64) WT_ARCH=x86_64-linux ;; \
+      aarch64|arm64) WT_ARCH=aarch64-linux ;; \
+      *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;; \
+    esac; \
+    curl -sSL -o wasm-tools.tar.gz \
+      "https://github.com/bytecodealliance/wasm-tools/releases/download/v${WASM_TOOLS_VERSION}/wasm-tools-${WASM_TOOLS_VERSION}-${WT_ARCH}.tar.gz"; \
+    tar -xzf wasm-tools.tar.gz; \
+    cp "wasm-tools-${WASM_TOOLS_VERSION}-${WT_ARCH}/wasm-tools" /usr/local/bin/; \
+    wasm-tools --version; \
+    curl -sSL -o wasi_snapshot_preview1.command.wasm \
+      "https://github.com/bytecodealliance/wasmtime/releases/download/v${ADAPTER_WASMTIME_VERSION}/wasi_snapshot_preview1.command.wasm"
+
+COPY --from=kotlin-build \
+     /src/build/compileSync/wasmWasi/main/productionExecutable/optimized/kotlin-richards.wasm \
+     core.wasm
+COPY wit ./wit
+
+# The Kotlin WASI executable exports `_initialize` (a WASI "reactor" entry that
+# runs `fun main()`). Rename that export to `_start` so the preview1 *command*
+# adapter binds it to `wasi:cli/run`'s `run`, which is how sightglass invokes a
+# component benchmark. Then embed the world describing the `bench` import and
+# create the component (the adapter supplies the WASI imports + `run` export).
+RUN set -eux; \
+    wasm-tools print core.wasm \
+      | sed -E 's/\(export "_initialize" \(func /(export "_start" (func /' \
+      | wasm-tools parse -o command.wasm; \
+    wasm-tools component embed wit/bench.wit --world richards command.wasm -o embedded.wasm; \
+    wasm-tools component new embedded.wasm \
+      --adapt wasi_snapshot_preview1=wasi_snapshot_preview1.command.wasm \
+      -o kotlin-richards.wasm; \
+    wasm-tools validate --features all kotlin-richards.wasm; \
+    mkdir -p /benchmark; \
+    cp kotlin-richards.wasm /benchmark/
+# We output the Wasm file to the `/benchmark` directory, where the client
+# expects it.

benchmarks/kotlin-richards/README.md

@@ -0,0 +1,29 @@
+# kotlin-richards
+
+A Kotlin port of the classic Richards benchmark: a simulation of an operating
+system's scheduler. This version is a Wasm component produced via Kotlin/Wasm's
+`wasmWasi` target and `wasm-tools component new`.
+
+Richards is an integer/pointer-heavy workload built around linked lists of
+"packets" routed between cooperating tasks (idle, worker, handler, device). It
+exercises object allocation, virtual dispatch, and pointer chasing, and here it
+also exercises the WasmGC, typed-function-references, and exception-handling
+proposals that Kotlin/Wasm relies on.
+
+## Origin
+
+The port follows the JavaScript implementation distributed with JetStream 2:
+
+* Source: <https://browserbench.org/JetStream2.0/Octane/richards.js>
+  (also published in Google's Octane benchmark suite as `richards.js`).
+
+That JavaScript is itself a port of Martin Richards' original BCPL benchmark
+(<https://www.cl.cam.ac.uk/~mr10/Bench.html>). `Richards.kt` is a faithful,
+class-for-class translation of the JavaScript.
+
+### License
+
+The original JavaScript `richards.js` is copyright 2006-2008 the V8 project
+authors and is distributed under a BSD 3-clause license (see the header of the
+source linked above). This Kotlin port is made available under the same licenses
+as the rest of sightglass (Apache-2.0 WITH LLVM-exception, or MIT).

benchmarks/kotlin-richards/build.gradle.kts

@@ -0,0 +1,31 @@
+@file:OptIn(org.jetbrains.kotlin.gradle.ExperimentalWasmDsl::class)
+
+import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask
+
+plugins {
+    kotlin("multiplatform") version "2.3.0"
+}
+
+repositories {
+    mavenCentral()
+}
+
+kotlin {
+    wasmWasi {
+        // A WASI "command": emits a core module exporting `_start` (whose body is
+        // `fun main()`) plus linear `memory`, ready for the preview1->component
+        // command adapter.
+        nodejs()
+        binaries.executable()
+    }
+}
+
+tasks.withType<KotlinCompilationTask<*>>().configureEach {
+    compilerOptions.freeCompilerArgs.addAll(
+        // Opt in to the experimental host-import and unsafe-linear-memory APIs we
+        // use for the `bench` hooks and WASI file reading.
+        "-opt-in=kotlin.wasm.ExperimentalWasmInterop",
+        "-opt-in=kotlin.wasm.unsafe.UnsafeWasmMemoryApi",
+    )
+}

benchmarks/kotlin-richards/default.input

@@ -0,0 +1 @@
+120

benchmarks/kotlin-richards/gradle.properties

@@ -0,0 +1,2 @@
+kotlin.code.style=official
+org.gradle.jvmargs=-Xmx2g

benchmarks/kotlin-richards/kotlin-richards.stdout.expected

@@ -0,0 +1,4 @@
+[kotlin-richards] iterations: 120
+[kotlin-richards] queue count: 278640
+[kotlin-richards] hold count: 111360
+[kotlin-richards] verified

benchmarks/kotlin-richards/settings.gradle.kts

@@ -0,0 +1,8 @@
+rootProject.name = "kotlin-richards"
+
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+        mavenCentral()
+    }
+}

benchmarks/kotlin-richards/src/wasmWasiMain/kotlin/Main.kt

@@ -0,0 +1,147 @@
+import kotlin.wasm.WasmImport
+import kotlin.wasm.unsafe.withScopedMemoryAllocator
+
+// ===========================================================================
+// Sightglass benchmarking hooks.
+//
+// The harness records execution time / performance counters between these two
+// calls. As core-module imports they are `(import "bench" "start" (func))` and
+// `(import "bench" "end" (func))`; after `wasm-tools component new` they surface
+// as an imported `bench` instance exporting `start`/`end`, which is exactly what
+// the sightglass component runner supplies.
+// ===========================================================================
+
+@WasmImport("bench", "start")
+private external fun benchStart()
+
+@WasmImport("bench", "end")
+private external fun benchEnd()
+
+// ===========================================================================
+// Minimal WASI preview1 bindings for reading the workload file.
+//
+// Kotlin/Wasm's `wasmWasi` standard library has no general file API, so we bind
+// the handful of `wasi_snapshot_preview1` functions we need directly. The
+// WASI-preview1-to-component adapter lowers these to `wasi:filesystem` when the
+// component is created.
+// ===========================================================================
+
+@WasmImport("wasi_snapshot_preview1", "path_open")
+private external fun pathOpen(
+    fd: Int,
+    dirflags: Int,
+    pathPtr: Int,
+    pathLen: Int,
+    oflags: Int,
+    rightsBase: Long,
+    rightsInheriting: Long,
+    fdflags: Int,
+    resultFdPtr: Int,
+): Int
+
+@WasmImport("wasi_snapshot_preview1", "fd_read")
+private external fun fdRead(fd: Int, iovsPtr: Int, iovsLen: Int, nreadPtr: Int): Int
+
+@WasmImport("wasi_snapshot_preview1", "fd_close")
+private external fun fdClose(fd: Int): Int
+
+/**
+ * The benchmark's working directory is preopened by the harness as the first
+ * (and only) WASI preopen, which is conventionally file descriptor 3.
+ */
+private const val PREOPEN_FD = 3
+
+/** `__WASI_RIGHTS_FD_READ`: the right to `fd_read` the opened file. */
+private const val RIGHTS_FD_READ = 2L
+
+/**
+ * Open [path] in the preopened working directory, read it, and return the
+ * leading decimal integer it contains (the Richards iteration count).
+ */
+private fun readIterationCount(path: String): Int = withScopedMemoryAllocator { allocator ->
+    // Write the path bytes into linear memory for `path_open`.
+    val pathBytes = path.encodeToByteArray()
+    val pathPtr = allocator.allocate(pathBytes.size)
+    for (i in pathBytes.indices) {
+        (pathPtr + i).storeByte(pathBytes[i])
+    }
+
+    // Open the file relative to the preopened directory.
+    val fdResult = allocator.allocate(4)
+    val openRc = pathOpen(
+        PREOPEN_FD,
+        0,
+        pathPtr.address.toInt(),
+        pathBytes.size,
+        0,
+        RIGHTS_FD_READ,
+        0L,
+        0,
+        fdResult.address.toInt(),
+    )
+    check(openRc == 0) { "path_open(\"$path\") failed with errno $openRc" }
+    val fd = fdResult.loadInt()
+
+    // Read the file into a buffer via a single iovec { ptr, len }.
+    val capacity = 64
+    val buffer = allocator.allocate(capacity)
+    val iovec = allocator.allocate(8)
+    iovec.storeInt(buffer.address.toInt())
+    (iovec + 4).storeInt(capacity)
+    val nreadPtr = allocator.allocate(4)
+    val readRc = fdRead(fd, iovec.address.toInt(), 1, nreadPtr.address.toInt())
+    check(readRc == 0) { "fd_read failed with errno $readRc" }
+    val bytesRead = nreadPtr.loadInt()
+    fdClose(fd)
+
+    // Parse the leading decimal integer out of the bytes read.
+    var value = 0
+    var sawDigit = false
+    for (i in 0 until bytesRead) {
+        val c = (buffer + i).loadByte().toInt()
+        if (c in '0'.code..'9'.code) {
+            value = value * 10 + (c - '0'.code)
+            sawDigit = true
+        } else if (sawDigit) {
+            break
+        }
+    }
+    check(sawDigit) { "no integer found in \"$path\"" }
+    value
+}
+
+fun main() {
+    // 1. Read the workload (the number of Richards iterations) from the
+    //    filesystem via WASI.
+    val iterations = readIterationCount("default.input")
+
+    // 2. Begin recording.
+    benchStart()
+
+    // 3. The main work: run the Richards scheduler simulation `iterations`
+    //    times, accumulating the packet-queue and task-hold counts each run
+    //    produces.
+    var totalQueueCount = 0L
+    var totalHoldCount = 0L
+    var allValid = true
+    for (i in 0 until iterations) {
+        val scheduler = runRichards()
+        totalQueueCount += scheduler.queueCount
+        totalHoldCount += scheduler.holdCount
+        if (scheduler.queueCount != EXPECTED_QUEUE_COUNT ||
+            scheduler.holdCount != EXPECTED_HOLD_COUNT
+        ) {
+            allValid = false
+        }
+    }
+
+    // 4. Stop recording.
+    benchEnd()
+
+    // 5. Print results derived from the work so an optimizing compiler cannot
+    //    elide the simulation.
+    println("[kotlin-richards] iterations: $iterations")
+    println("[kotlin-richards] queue count: $totalQueueCount")
+    println("[kotlin-richards] hold count: $totalHoldCount")
+    println("[kotlin-richards] " + if (allValid) "verified" else "INVALID")
+}