feat(jco): detect older wasi:http version for older componentizejs

Author: vados-cosmonic

crates/wasm-tools-component/src/lib.rs

@@ -17,7 +17,7 @@ mod bindings {
 
 use bindings::exports::local::wasm_tools::tools::{
     EmbedOpts, EnabledFeatureSet, Guest, InterfaceMetadata, ModuleMetaType, ModuleMetadata,
-    ProducersFields, SemverVersion, StringEncoding, WitMetadata,
+    ProducersFields, SemverVersion, StringEncoding, WitMetadata, WitSpecifier,
 };
 
 struct WasmToolsJs;
@@ -73,32 +73,39 @@ impl Guest for WasmToolsJs {
     }
 
     fn component_wit_metadata_for_world(
-        binary: Vec<u8>,
+        wit: WitSpecifier,
         maybe_world_name: Option<String>,
     ) -> Result<WitMetadata, String> {
-        let decoded = wit_component::decode(&binary)
-            .map_err(|e| format!("Failed to decode wit component\n{e:?}"))?;
-
-        let (resolve, world_id) = match &decoded {
-            DecodedWasm::WitPackage(_, _) => panic!("Unexpected wit package"),
-            DecodedWasm::Component(resolve, world) => (resolve, world),
-        };
+        let mut resolve = Resolve::default();
 
-        let world = match maybe_world_name {
-            Some(world_name) => {
+        // Resolve package IDs & build Resolve for WIT
+        let package_ids = match wit {
+            WitSpecifier::Source(source) => {
+                let path = PathBuf::from("component.wit");
                 resolve
-                    .worlds
-                    .iter()
-                    .find(|w| w.1.name == world_name)
-                    .ok_or_else(|| String::from("failed to find wold with given name"))?
-                    .1
+                    .push_str(&path, &source)
+                    .map_err(|e| e.to_string())?
+            }
+            WitSpecifier::Path(path) => {
+                let wit_path_meta =
+                    metadata(&path).map_err(|e| format!("failed to get path metadata: {e}"))?;
+                if wit_path_meta.is_file() {
+                    resolve.push_file(path).map_err(|e| e.to_string())?
+                } else {
+                    resolve.push_dir(path).map_err(|e| e.to_string())?.0
+                }
             }
-            None => resolve
-                .worlds
-                .get(*world_id)
-                .ok_or_else(|| String::from("failed to find package id"))?,
         };
 
+        let world_id = resolve
+            .select_world(&[package_ids], maybe_world_name.as_deref())
+            .map_err(|e| e.to_string())?;
+        let world = resolve
+            .worlds
+            .get(world_id)
+            .ok_or_else(|| String::from("failed to find package id"))?;
+
+        // Gather imports
         let mut imports = Vec::new();
         for (import_key, import_item) in world.imports.iter() {
             let WorldItem::Interface { id, .. } = import_item else {
@@ -140,6 +147,7 @@ impl Guest for WasmToolsJs {
             });
         }
 
+        // Gather exports
         let mut exports = Vec::new();
         for (export_key, export_item) in world.exports.iter() {
             let WorldItem::Interface { id, .. } = export_item else {

crates/wasm-tools-component/wit/wasm-tools.wit

@@ -43,10 +43,21 @@ interface tools {
       exports: list<interface-metadata>,
   }
 
+  /// Way to specify WIT
+  variant wit-specifier {
+      /// WIT raw source
+      source(string),
+      /// Path to wit file or directory
+      path(string),
+  }
+
   /// Extract metadata from a WIT interface for a component for a given world
   ///
   /// If no world is specified, the root world is used
-  component-wit-metadata-for-world: func(wit-bytes: list<u8>, world-name: option<string>) -> result<wit-metadata, string>;
+  component-wit-metadata-for-world: func(
+    wit: wit-specifier,
+    world-name: option<string>
+  ) -> result<wit-metadata, string>;
 
   type producers-fields = list<tuple<string, list<tuple<string, string>>>>;
 

packages/jco/package.json

@@ -75,7 +75,6 @@
     "dependencies": {
         "@bytecodealliance/componentize-js": "^0.20.0",
         "@bytecodealliance/componentize-js-0-19-3": "npm:@bytecodealliance/componentize-js@^0.19.3",
-        "@bytecodealliance/componentize-js": "^0.20.0",
         "@bytecodealliance/preview2-shim": "^0.17.9",
         "binaryen": "^123.0.0",
         "commander": "^14",

packages/jco/src/cmd/componentize.js

@@ -1,5 +1,8 @@
 import { stat, readFile, writeFile } from "node:fs/promises";
 import { resolve, basename } from "node:path";
+
+import * as wasmToolsComponent from "../../obj/wasm-tools.js";
+
 import { styleText } from "../common.js";
 
 /** All features that can be enabled/disabled */
@@ -8,19 +11,60 @@ const ALL_FEATURES = ["clocks", "http", "random", "stdio", "fetch-event"];
 /** Features that should be used for --debug mode */
 const DEBUG_FEATURES = ["stdio"];
 
-export async function componentize(jsSource, opts) {
-    const { componentize: componentizeFn } = await eval('import("@bytecodealliance/componentize-js")');
+/**
+ * Detect whether the WIT of a given component contains an older version of
+ * `wasi:http` which necessitates an older version of `componentize-js`
+ *
+ * @param {string} witPath
+ * @returns bool
+ */
+async function usesOlderWasiHTTP(witPath, worldName) {
+    await wasmToolsComponent.$init;
+
+    const worldMetadata = wasmToolsComponent.tools.componentWitMetadataForWorld(
+        { tag: "path", val: witPath },
+        worldName ?? null,
+    );
 
+    // Check if the an old `wasi:http/incoming-handler` version is exported
+    const exportsOldIncomingHandler = worldMetadata.exports.some((iface) => {
+        return (
+            iface.namespace === "wasi" &&
+            iface.package === "http" &&
+            iface.interface === "incoming-handler" &&
+            iface.version !== null &&
+            iface.version.major === 0n &&
+            iface.version.minor < 3n &&
+            iface.version.patch < 10n
+        );
+    });
+
+    return exportsOldIncomingHandler;
+}
+export async function componentize(jsSource, opts) {
     const { disableFeatures, enableFeatures } = calculateFeatureSet(opts);
 
     const source = await readFile(jsSource, "utf8");
-
     const witPath = resolve(opts.wit);
     const sourceName = basename(jsSource);
 
+    // Load an older version of componentize-js if we detect an older version of WASI HTTP in use
+    // as the version that is usable is baked into the StarlingMonkey version provided by a given version
+    // of componentize-js
+    let componentizeJSModule;
+    const useOldComponentizeJS = await usesOlderWasiHTTP(witPath, opts.worldName);
+    if (useOldComponentizeJS) {
+        // NOTE: if we were to use a version of componentize-js 0.20.0 or newer here,
+        // the build would fail, as newer versions do not support wasi:http < 0.2.10
+        // for fetch.
+        componentizeJSModule = await eval('import("@bytecodealliance/componentize-js-0-19-3")');
+    } else {
+        componentizeJSModule = await eval('import("@bytecodealliance/componentize-js")');
+    }
+
     let component;
     try {
-        const result = await componentizeFn(source, {
+        const result = await componentizeJSModule.componentize(source, {
             enableAot: opts.aot,
             aotMinStackSizeBytes: opts.aotMinStackSizeBytes,
             wevalBin: opts.wevalBin,
@@ -41,7 +85,7 @@ export async function componentize(jsSource, opts) {
             },
         });
         if (result.debug) {
-            console.error(`${styleText("cyan", "DEBUG")} Debug output\n${JSON.stringify(debug, null, 2)}\n`);
+            console.error(`${styleText("cyan", "DEBUG")} Debug output\n${JSON.stringify(result.debug, null, 2)}\n`);
         }
 
         component = result.component;

packages/jco/test/common.js

@@ -22,6 +22,9 @@ export const JCO_JS_PATH = fileURLToPath(new URL("../src/jco.js", import.meta.ur
 /** Path to fixture components */
 export const COMPONENT_FIXTURES_DIR = fileURLToPath(new URL("./fixtures/components", import.meta.url));
 
+/** Path to JS for fixture components */
+export const COMPONENT_JS_FIXTURES_DIR = fileURLToPath(new URL("./fixtures/componentize", import.meta.url));
+
 /** Path to p3 related fixture components */
 export const P3_COMPONENT_FIXTURES_DIR = join(COMPONENT_FIXTURES_DIR, "p3");
 

packages/jco/test/componentize.js

@@ -0,0 +1,32 @@
+/* global Buffer */
+import { join } from "node:path";
+
+import { suite, test, assert } from "vitest";
+import { COMPONENT_JS_FIXTURES_DIR } from "./common.js";
+import { exec, getTmpDir, jcoPath } from "./helpers.js";
+
+// NOTE: we test componentization with the jco CLI to avoid
+// triggering errors for the the eval(import) call(s) in cmd/componentize.js
+//
+// TODO(breaking): once jco-transpile is established as a separate package and
+// used widely, we can switch to regular dynamic imports, as componentize-js
+// versions are real dependencies now.
+suite("componentize", () => {
+    test.concurrent("detect older wasi:http", async () => {
+        const jsPath = join(COMPONENT_JS_FIXTURES_DIR, "wasi-http-detection-old/component.js");
+        const witPath = join(COMPONENT_JS_FIXTURES_DIR, "wasi-http-detection-old/wit");
+        const outputDir = await getTmpDir();
+        const outputPath = join(outputDir, "component.wasm");
+        const { stderr } = await exec(jcoPath, "componentize", jsPath, "-w", witPath, "-o", outputPath);
+        assert.strictEqual(stderr, "");
+    });
+
+    test.concurrent("detect newer wasi:http", async () => {
+        const jsPath = join(COMPONENT_JS_FIXTURES_DIR, "wasi-http-detection-new/component.js");
+        const witPath = join(COMPONENT_JS_FIXTURES_DIR, "wasi-http-detection-new/wit");
+        const outputDir = await getTmpDir();
+        const outputPath = join(outputDir, "component.wasm");
+        const { stderr } = await exec(jcoPath, "componentize", jsPath, "-w", witPath, "-o", outputPath);
+        assert.strictEqual(stderr, "");
+    });
+});

packages/jco/test/fixtures/componentize/wasi-http-detection-new/component.js

@@ -0,0 +1,7 @@
+addEventListener("fetch", (event) =>
+    event.respondWith(
+        (async () => {
+            return new Response("Hello World");
+        })(),
+    ),
+);

packages/jco/test/fixtures/componentize/wasi-http-detection-new/wit/component.wit

@@ -0,0 +1,5 @@
+package jco-fixtures:wasi-http-detection;
+
+world use-new {
+    export wasi:http/incoming-handler@0.2.10;
+}

packages/jco/test/fixtures/componentize/wasi-http-detection-new/wit/deps/wasi-cli-0.2.10/package.wit

@@ -0,0 +1,20 @@
+package wasi:cli@0.2.10;
+
+interface stdout {
+  use wasi:io/streams@0.2.10.{output-stream};
+
+  get-stdout: func() -> output-stream;
+}
+
+interface stderr {
+  use wasi:io/streams@0.2.10.{output-stream};
+
+  get-stderr: func() -> output-stream;
+}
+
+interface stdin {
+  use wasi:io/streams@0.2.10.{input-stream};
+
+  get-stdin: func() -> input-stream;
+}
+

packages/jco/test/fixtures/componentize/wasi-http-detection-new/wit/deps/wasi-clocks-0.2.10/package.wit

@@ -0,0 +1,29 @@
+package wasi:clocks@0.2.10;
+
+interface monotonic-clock {
+  use wasi:io/poll@0.2.10.{pollable};
+
+  type instant = u64;
+
+  type duration = u64;
+
+  now: func() -> instant;
+
+  resolution: func() -> duration;
+
+  subscribe-instant: func(when: instant) -> pollable;
+
+  subscribe-duration: func(when: duration) -> pollable;
+}
+
+interface wall-clock {
+  record datetime {
+    seconds: u64,
+    nanoseconds: u32,
+  }
+
+  now: func() -> datetime;
+
+  resolution: func() -> datetime;
+}
+