diff --git a/.github/workflows/phone.yml b/.github/workflows/phone.yml index ed0d3db38..67968d100 100644 --- a/.github/workflows/phone.yml +++ b/.github/workflows/phone.yml @@ -184,6 +184,7 @@ jobs: timeout-minutes: 60 run: | cargo test --locked -p octosense-wasm-host --no-run + cargo test --locked -p octosense-wasm-host --features pulley --no-run cargo test --locked -p makepad-widgets --no-run cargo test --locked -p makepad-script-std --no-run - name: Test Wasm and runtime policy @@ -196,7 +197,9 @@ jobs: # Apps' WebAssembly functions: the runtime's own tests. The `wasm` # service's (fresh state per call; resource, cancellation and # admission bounds) run with the shell's above: `mobile-apps` - # includes `wasm-functions`. + # includes `wasm-functions`. Then the same tests in Pulley, the + # interpreter OpenHarmony and iOS use (ADR 0014). cargo test --locked -p octosense-wasm-host + OCTOSENSE_WASM_PULLEY=1 cargo test --locked -p octosense-wasm-host --features pulley cargo test --locked -p makepad-widgets splash_policy cargo test --locked -p makepad-script-std gate::tests diff --git a/.github/workflows/wasm-windows.yml b/.github/workflows/wasm-windows.yml new file mode 100644 index 000000000..e03bca931 --- /dev/null +++ b/.github/workflows/wasm-windows.yml @@ -0,0 +1,50 @@ +name: WebAssembly on Windows + +# Apps' own functions on Windows (ADR 0014): the runtime's tests +# (crates/wasm-host) and the shell compiled with the `wasm` service, which +# Windows desktop builds include. ci-local cannot run these (tools/ci_local.py, +# NOT_LOCAL), so a pull request that touches them merges on this run. + +on: + push: + branches: [main] + pull_request: + paths: + - 'crates/wasm-host/**' + - 'crates/shell/src/wasm_service.rs' + - 'crates/shell/build.rs' + - 'crates/shell/Cargo.toml' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/wasm-windows.yml' + +permissions: + contents: read + +concurrency: + group: wasm-windows-${{ github.ref }} + cancel-in-progress: true + +jobs: + windows: + runs-on: windows-2022 + timeout-minutes: 55 + env: + PYTHONUTF8: '1' + steps: + - name: Preserve runtime patch bytes + run: git config --global core.autocrlf false + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - uses: dtolnay/rust-toolchain@stable + - run: python tools/setup.py + - uses: Swatinem/rust-cache@v2 + with: + shared-key: wasm-windows + cache-on-failure: true + - name: The runtime's tests + run: cargo test --locked -p octosense-wasm-host + - name: The shell with the wasm service + run: cargo check --locked -p octosense-shell --features wasm-functions diff --git a/Cargo.lock b/Cargo.lock index f0da8242b..30ced5e7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11383,7 +11383,7 @@ dependencies = [ [[package]] name = "octosense-app-contract" version = "1.10.0" -source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=28745c0f6fd71986820e9835183a3045f61baefc#28745c0f6fd71986820e9835183a3045f61baefc" +source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339#371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" dependencies = [ "blake3", "serde", @@ -11394,7 +11394,7 @@ dependencies = [ [[package]] name = "octosense-app-hub" version = "0.1.0" -source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=28745c0f6fd71986820e9835183a3045f61baefc#28745c0f6fd71986820e9835183a3045f61baefc" +source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339#371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" dependencies = [ "base64 0.22.1", "blake3", @@ -11414,12 +11414,13 @@ dependencies = [ "sigstore-types", "sigstore-verify", "ureq 3.4.2", + "wasmparser 0.258.3", ] [[package]] name = "octosense-app-hub-app" version = "0.1.0" -source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=28745c0f6fd71986820e9835183a3045f61baefc#28745c0f6fd71986820e9835183a3045f61baefc" +source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339#371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" dependencies = [ "makepad-app-module", "makepad-widgets", @@ -11447,7 +11448,7 @@ dependencies = [ [[package]] name = "octosense-app-policy" version = "0.1.0" -source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=28745c0f6fd71986820e9835183a3045f61baefc#28745c0f6fd71986820e9835183a3045f61baefc" +source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339#371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" dependencies = [ "makepad-widgets", "octosense-app-contract", @@ -11469,7 +11470,7 @@ dependencies = [ [[package]] name = "octosense-appstore" version = "0.1.0" -source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=28745c0f6fd71986820e9835183a3045f61baefc#28745c0f6fd71986820e9835183a3045f61baefc" +source = "git+https://github.com/OctoSense-org/OctoSense-App-Hub?rev=371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339#371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" dependencies = [ "makepad-app-module", "makepad-widgets", @@ -11984,12 +11985,16 @@ name = "octosense-wasm-host" version = "0.1.0" dependencies = [ "base64 0.22.1", + "bytes", + "http", + "http-body-util", "libc", "serde_json", "sha2 0.10.9", "wasmlab-functions", "wasmtime", "wasmtime-wasi", + "wasmtime-wasi-http", "wat", ] @@ -19037,6 +19042,30 @@ dependencies = [ "winx", ] +[[package]] +name = "wasmtime-wasi-http" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4cd327c70de1309b0c04d49df3633a2ffeae9b29b526d421320b565e198175" +dependencies = [ + "async-trait", + "bytes", + "futures", + "http", + "http-body", + "http-body-util", + "hyper", + "pin-project-lite", + "rustls", + "tokio", + "tokio-rustls", + "tracing", + "wasmtime", + "wasmtime-wasi", + "wasmtime-wasi-io", + "webpki-roots 1.0.9", +] + [[package]] name = "wasmtime-wasi-io" version = "49.0.2" diff --git a/Cargo.toml b/Cargo.toml index 9dfaf2d45..8b8c6f806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,7 +91,7 @@ octosense-reference = { path = "apps/reference" } makepad-sheets = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2", default-features = false } makepad-terminal = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2" } octosense-appcard = { path = "apps/appcard/module" } -octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "28745c0f6fd71986820e9835183a3045f61baefc" } +octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" } makepad-calculator = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2", default-features = false } makepad-clock = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2", default-features = false } makepad-notes = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2", default-features = false } @@ -130,12 +130,12 @@ octoscript-makepad = { git = "https://github.com/OctoSense-org/Octoscript-Makepa octoscript-node = { git = "https://github.com/OctoSense-org/Octoscript-Makepad.git", rev = "33dea2f1f3ad3f1346a219aa8cf6e91b31361e23" } # --- App Hub's host service registry the Mail and llm services register # into; the same revision as App Hub's app crate (a native app, above). -octosense-appstore = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "28745c0f6fd71986820e9835183a3045f61baefc" } +octosense-appstore = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" } # The app policy (resolution, isolate settings, the Splash adapter) and the # Hub client the Card runner resolves an app's policy with; the shell's # glance tiles run cards under it (crates/shell/src/glance_card.rs). -octosense-app-policy = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "28745c0f6fd71986820e9835183a3045f61baefc" } -octosense-app-hub = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "28745c0f6fd71986820e9835183a3045f61baefc" } +octosense-app-policy = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" } +octosense-app-hub = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" } # The app contract (ADR 0005), from crates.io by version: App Hub, the shell # and Rinx all depend on `octosense-app-contract = "1"`, so Cargo links ONE # 1.x for the whole build (Cargo.lock pins the exact version OctoSense ships) @@ -313,9 +313,11 @@ log = "0.4" # Developers' own functions compiled to WebAssembly, run for their app with # a deadline and a memory cap (crates/wasm-host, ADR 0011). Cranelift compiles; # no GC or threads. Components (ADR 0014) add the component model and a WASI -# 0.2 subset scoped to the app: `wasmtime-wasi`'s `p2` only. +# 0.2 subset scoped to the app: `wasmtime-wasi`'s `p2` only, and outgoing +# `wasi:http` to the app's own network hosts (rustls with ring, as the shell's). wasmtime = { version = "49.0.2", default-features = false, features = ["cranelift", "runtime", "std", "parallel-compilation", "component-model"] } wasmtime-wasi = { version = "49.0.2", default-features = false, features = ["p2"] } +wasmtime-wasi-http = { version = "49.0.2", default-features = false, features = ["p2", "default-send-request"] } # Workspace-wide clippy policy for the crates that opt in (`[lints] # workspace = true`, AppCard's): lints that are real but tracked elsewhere. @@ -435,7 +437,7 @@ ruma = { git = "https://github.com/project-robius/ruma.git", branch = "tsp" } # -i octos-core` still sees one octos source. [patch.crates-io] # Keep the contract identical to the pinned SDK for this release candidate. -octosense-app-contract = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "28745c0f6fd71986820e9835183a3045f61baefc" } +octosense-app-contract = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339" } nix = { git = "https://github.com/octos-org/octos.git", rev = "18fcd3f16e527d2b601d7ae244f056fb711bb6b8" } # pdfcraft's bootstrap PDF stack: its workspace patches these five diff --git a/crates/shell/Cargo.toml b/crates/shell/Cargo.toml index f1b84e318..f372e6a08 100644 --- a/crates/shell/Cargo.toml +++ b/crates/shell/Cargo.toml @@ -151,11 +151,12 @@ octosense-calendar-service = { workspace = true } octosense-ai-host = { workspace = true, features = ["llm"] } # Apps' own WebAssembly functions (the `wasm` service, ADR 0011), compiled by -# Cranelift: only where an app may generate code and the runtime has been -# checked. Windows (not yet checked), iOS (no code generation for apps), -# OpenHarmony (its policy is unknown) and the browser leave it out; build.rs -# sets `wasm_functions` for the same targets. -[target.'cfg(any(target_os = "macos", target_os = "android", all(target_os = "linux", not(target_env = "ohos"))))'.dependencies] +# Cranelift where an app may generate code, and on OpenHarmony, until its +# policy is known, run in Pulley, Wasmtime's interpreter (ADR 0014; the +# runtime selects it there). Windows runs its tests in CI +# (.github/workflows/wasm-windows.yml). iOS (not yet built with it) and the +# browser leave it out; build.rs sets `wasm_functions` for the same targets. +[target.'cfg(any(target_os = "macos", target_os = "windows", target_os = "android", target_os = "linux"))'.dependencies] octosense-wasm-host = { workspace = true, optional = true } [features] @@ -195,7 +196,7 @@ dev-mode = [] # The `wasm` host service (ADR 0011), which runs an app's own # WebAssembly functions (crates/wasm-host). On in every standard shell # build (tools/native_apps.py's BASE_DEFAULT); it runs where build.rs sets -# `wasm_functions`: macOS, Linux and Android. +# `wasm_functions`: macOS, Windows, Linux, Android and OpenHarmony. wasm-functions = ["app-hub", "dep:octosense-wasm-host"] # The feature's former name. wasm-lab = ["wasm-functions"] diff --git a/crates/shell/build.rs b/crates/shell/build.rs index b9a57230b..b158882b4 100644 --- a/crates/shell/build.rs +++ b/crates/shell/build.rs @@ -8,7 +8,8 @@ //! `cfg!(native_mobile)`, never as the feature or the targets directly. //! //! `wasm_functions` marks a build that runs the `wasm` service (ADR 0011): -//! feature `wasm-functions` on macOS, Linux or Android, the targets +//! feature `wasm-functions` on macOS, Windows, Linux, Android or OpenHarmony +//! (there in Pulley, Wasmtime's interpreter: ADR 0014), the targets //! Cargo.toml links its runtime for. Code reads `#[cfg(wasm_functions)]`. //! //! `dev_mode` marks a development build (ADR 0004 §13, `src/dev_mode.rs`): @@ -35,7 +36,7 @@ fn main() { println!("cargo:rustc-cfg=mobile_only"); } println!("cargo:rustc-check-cfg=cfg(wasm_functions)"); - let wasm_target = target_os == "macos" || target_os == "android" || (target_os == "linux" && target_env != "ohos"); + let wasm_target = matches!(target_os.as_str(), "macos" | "windows" | "android" | "linux"); if wasm_target && std::env::var_os("CARGO_FEATURE_WASM_FUNCTIONS").is_some() { println!("cargo:rustc-cfg=wasm_functions"); } diff --git a/crates/shell/src/host_tools/mod.rs b/crates/shell/src/host_tools/mod.rs index 53fe65497..0ae73914d 100644 --- a/crates/shell/src/host_tools/mod.rs +++ b/crates/shell/src/host_tools/mod.rs @@ -121,6 +121,9 @@ pub fn pump() { let Ok(_pump) = PUMP.try_lock() else { return }; #[cfg(any(feature = "app-hub", native_mobile))] script_apps::poll(); + // Components' calls to their apps' host services (ADR 0014). + #[cfg(wasm_functions)] + crate::wasm_service::pump_host_calls(); for _ in 0..8 { let events = std::mem::take(&mut *INBOX.lock().unwrap_or_else(|e| e.into_inner())); if events.is_empty() { diff --git a/crates/shell/src/lib.rs b/crates/shell/src/lib.rs index 80c140bec..cb9cb2f58 100644 --- a/crates/shell/src/lib.rs +++ b/crates/shell/src/lib.rs @@ -6326,6 +6326,9 @@ impl MatchEvent for App { self.installed_app_changed(cx, &id); // Its tools, grants and kernel tools, as installed (ADR 0004 §7). host_tools::script_app_installed(&id); + // Its own functions, compiled before its first call (ADR 0014). + #[cfg(wasm_functions)] + wasm_service::warm(&id); } self.drain_hub(cx); self.drain_client_lines(cx); diff --git a/crates/shell/src/wasm_service.rs b/crates/shell/src/wasm_service.rs index 79bbed822..83dc22633 100644 --- a/crates/shell/src/wasm_service.rs +++ b/crates/shell/src/wasm_service.rs @@ -23,8 +23,21 @@ //! Admission, cancellation and deadline are checked before execution and delivery; //! an update or withdrawal invalidates loaded Programs. Workers exit when idle. //! The host-only disk cache holds compiled code keyed by module digest, not data. +//! +//! **Components** (ADR 0014). A `fns/*.wasm` may also be a WebAssembly +//! component, built with plain `cargo build --target wasm32-wasip2`. Each of +//! its exported functions is callable as `wasm.` with JSON (snake_case +//! names; `octosense_wasm_host::component` maps the values). Unlike a +//! module's, a component's instance lives on in the worker between calls, +//! so it can keep a document or a cache; a trap, a deadline, a changed grant +//! or the worker's exit (a minute without calls) ends it. Its filesystem is +//! the app's storage, the jail its script's `fs.*` sees, decided per call by +//! the same rules as an engine's ([`crate::host_tools::areas::app_area`]): +//! none without the `storage` capability or a signed-in account, and what +//! is left of the quota is what a call may add (a write past it fails inside +//! the component, as a full disk). -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -33,20 +46,162 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use octosense_appstore::services::{HostService, Replier, ServiceCall, ServiceHost}; +use octosense_wasm_host::component::{ + self, ComponentInstance, ComponentProgram, Grants, HostCalls, +}; use octosense_wasm_host::{Limits, Program, Runtime}; use serde_json::{json, Value}; +use crate::host_tools::areas::{self, AreaEnv}; use crate::host_tools::script_apps; const MAX_MODULES: usize = 8; const MAX_WORKERS: usize = 4; const MAX_QUEUED_PER_APP: usize = 4; -const MAX_INPUT_BYTES: usize = 1 << 20; -const MAX_BUFFERED_BYTES: usize = 16 << 20; +const MAX_INPUT_BYTES: usize = 8 << 20; +const MAX_BUFFERED_BYTES: usize = 32 << 20; const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const WORKER_IDLE: Duration = Duration::from_secs(5); +/// How long a worker that holds a component's live instance waits for the +/// next call before it exits (and the instance with it). +const COMPONENT_IDLE: Duration = Duration::from_secs(60); + +/// Where an app's component works: the shell's own rules, or a test's. +static AREA_ENV: Mutex>> = Mutex::new(None); + +fn area_env() -> Arc { + AREA_ENV + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get_or_insert_with(areas::shell) + .clone() +} static RUNTIME: OnceLock> = OnceLock::new(); + +// ------------------------------------------------- a component's host calls + +/// A component's call to one of its app's host services (`octosense:host`), +/// waiting for the UI thread ([`pump_host_calls`]), where its app's script +/// would make it. +struct HostCall { + call: ServiceCall, + deadline: Instant, + done: SyncSender>, +} + +static HOST_CALLS: Mutex> = Mutex::new(Vec::new()); +/// Dispatched calls by their request key, until answered or past their +/// deadline. +type Waiting = HashMap>, Instant)>; +static HOST_WAITING: Mutex> = Mutex::new(None); +/// Component calls are dispatched under keys of their own, apart from any +/// isolate's. +static NEXT_HOST_KEY: AtomicUsize = AtomicUsize::new(1 << 50); + +/// An app's host services as its components reach them: the families its +/// admitted manifest grants (a system app's own namespace too), dispatched +/// on the UI thread as its script's `host.request` would be, but never with +/// a sheet or a prompt, so only the methods a background surface may call. +struct AppHostCalls { + app: String, + host_dir: PathBuf, + families: BTreeSet, +} + +impl HostCalls for AppHostCalls { + fn request(&self, service: &str, args: &str, deadline: Instant) -> Result { + let family = service.split('.').next().unwrap_or(""); + // The app's worker is busy running this very call. + if family == "wasm" { + return Err( + "a component cannot call wasm.*: its app's functions are already running it".into(), + ); + } + let own = self + .app + .strip_prefix(octosense_appstore::system::SYSTEM_ID_PREFIX) + == Some(family); + if !self.families.contains(family) && !own { + return Err(format!( + "{} was not granted the {family} service, which {service} needs", + self.app + )); + } + let args: Value = serde_json::from_str(args) + .map_err(|e| format!("{service}: the arguments are not JSON: {e}"))?; + let (done, answer) = sync_channel(1); + HOST_CALLS + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(HostCall { + call: ServiceCall { + app_id: self.app.clone(), + service: service.to_string(), + args, + from_sheet: false, + may_prompt: false, + host_dir: self.host_dir.clone(), + }, + deadline, + done, + }); + makepad_widgets::makepad_platform::thread::SignalToUI::set_ui_signal(); + answer + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .unwrap_or_else(|_| { + Err(format!( + "{service} did not answer before the call's deadline" + )) + }) + } +} + +/// Dispatch the components' queued host calls and deliver their answers; +/// on the UI thread ([`crate::host_tools::pump`]). +pub fn pump_host_calls() { + struct NoSheets; + impl ServiceHost for NoSheets { + fn open_sheet(&mut self, _body: String) {} + fn close_sheet(&mut self) {} + } + let queued = std::mem::take(&mut *HOST_CALLS.lock().unwrap_or_else(|e| e.into_inner())); + let mut waiting = HOST_WAITING.lock().unwrap_or_else(|e| e.into_inner()); + let waiting = waiting.get_or_insert_with(HashMap::new); + for HostCall { + call, + deadline, + done, + } in queued + { + if Instant::now() >= deadline { + continue; + } + let key = NEXT_HOST_KEY.fetch_add(1, Ordering::Relaxed); + waiting.insert(key, (done, deadline)); + octosense_appstore::services::dispatch(call, key, 1, &mut NoSheets); + } + if waiting.is_empty() { + return; + } + let keys: Vec = waiting.keys().copied().collect(); + for (key, _, result) in octosense_appstore::services::take_replies_for(&keys) { + if let Some((done, _)) = waiting.remove(&key) { + let _ = done.try_send(result); + } + } + // A call whose component stopped waiting goes nowhere. + let now = Instant::now(); + let expired: Vec = waiting + .iter() + .filter(|(_, (_, deadline))| *deadline <= now) + .map(|(key, _)| *key) + .collect(); + for key in expired { + waiting.remove(&key); + octosense_appstore::services::cancel_heap(key); + } +} static WORKERS: Mutex>> = Mutex::new(None); static BUFFERED_BYTES: AtomicUsize = AtomicUsize::new(0); @@ -136,6 +291,8 @@ impl Admission { struct Job { method: String, input: Vec, + /// The request's arguments were a JSON string, and `input` is its text. + text: bool, host_dir: PathBuf, admission: Arc, deadline: Instant, @@ -189,7 +346,10 @@ fn input_bytes(args: &Value) -> Result, String> { impl Write for Bounded { fn write(&mut self, bytes: &[u8]) -> std::io::Result { if bytes.len() > MAX_INPUT_BYTES.saturating_sub(self.0.len()) { - return Err(std::io::Error::other("wasm input exceeds 1 MiB")); + return Err(std::io::Error::other(format!( + "wasm input exceeds {} MiB", + MAX_INPUT_BYTES >> 20 + ))); } self.0.extend_from_slice(bytes); Ok(bytes.len()) @@ -200,7 +360,7 @@ fn input_bytes(args: &Value) -> Result, String> { } if let Value::String(text) = args { if text.len() > MAX_INPUT_BYTES { - return Err("wasm input exceeds 1 MiB".into()); + return Err(format!("wasm input exceeds {} MiB", MAX_INPUT_BYTES >> 20)); } return Ok(text.as_bytes().to_vec()); } @@ -241,6 +401,7 @@ impl HostService for WasmService { let app = call.app_id.clone(); let mut job = Job { method: call.method().to_string(), + text: matches!(call.args, Value::String(_)), host_dir: call.host_dir, input, admission: Arc::default(), @@ -354,7 +515,14 @@ pub fn register() { fn work(app: String, jobs: Receiver, retired: &Arc) { let mut lab: Option = None; loop { - let job = match jobs.recv_timeout(WORKER_IDLE) { + // A component's live instance keeps its state: give the app longer + // to come back before it is dropped. + let idle = if lab.as_ref().is_some_and(Lab::holds_instances) { + COMPONENT_IDLE + } else { + WORKER_IDLE + }; + let job = match jobs.recv_timeout(idle) { Ok(job) => job, Err(_) => { // Serialize idle retirement with enqueue. A sender that won @@ -412,10 +580,13 @@ fn process_checked( *lab = None; return Err(error); } - let result = - lab.as_mut() - .unwrap() - .answer(&job.method, &job.input, job.deadline, job.reply.clone()); + let result = lab.as_mut().unwrap().answer( + &job.method, + &job.input, + job.text, + job.deadline, + job.reply.clone(), + ); before_delivery(); // A withdrawal, changed grant, update or tamper while code runs vetoes // its result and releases the old compiled-module references. @@ -442,6 +613,77 @@ struct Lab { owner: BTreeMap, stats: BTreeMap, admission: Admission, + /// Its host services, as its components reach them (`octosense:host`). + host_calls: Arc, +} + +/// Compile an installed or updated app's functions into the disk cache in +/// the background, so that its first call does not wait for Cranelift (ADR +/// 0014 phase 3; a phone takes about 0.4 s for a 433 KiB module). From its +/// admitted bundle only, and only with the `wasm` grant; one app at a time, +/// on a thread of its own, never in a worker's place. A call that comes +/// first compiles the same code itself: the cache takes either. +pub fn warm(app: &str) { + static QUEUE: Mutex<(Vec, bool)> = Mutex::new((Vec::new(), false)); + { + let mut queue = QUEUE.lock().unwrap_or_else(|e| e.into_inner()); + if !queue.0.iter().any(|queued| queued == app) { + queue.0.push(app.to_string()); + } + if std::mem::replace(&mut queue.1, true) { + return; + } + } + let spawned = std::thread::Builder::new() + .name("wasm-warm".into()) + .spawn(|| loop { + let next = { + let mut queue = QUEUE.lock().unwrap_or_else(|e| e.into_inner()); + if queue.0.is_empty() { + queue.1 = false; + return; + } + queue.0.remove(0) + }; + match warm_now(&next) { + Ok((0, _, _)) => {} + Ok((compiled, files, ms)) => makepad_widgets::log!( + "wasm {next}: compiled {compiled} of {files} function files ahead of its first call in {ms:.0} ms" + ), + Err(error) => makepad_widgets::log!("wasm {next}: not compiled ahead: {error}"), + } + }); + if spawned.is_err() { + QUEUE.lock().unwrap_or_else(|e| e.into_inner()).1 = false; + } +} + +/// [`warm`]'s work for one app: how many files it compiled, of how many, and +/// how long it took. +fn warm_now(app: &str) -> Result<(usize, usize, f64), String> { + let admission = Admission::current(app)?; + let runtime = runtime(&admission.root.join(".host"))?; + let started = Instant::now(); + let mut files: Vec = std::fs::read_dir(admission.bundle.join("fns")) + .map_err(|_| format!("{app}'s bundle has no fns directory"))? + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.extension().is_some_and(|ext| ext == "wasm") + && std::fs::symlink_metadata(path) + .is_ok_and(|m| m.is_file() && m.len() <= runtime.limits().module_bytes as u64) + }) + .collect(); + files.sort(); + files.truncate(MAX_MODULES); + let mut compiled = 0; + for path in &files { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + if runtime.precompile(&bytes).map_err(|e| e.to_string())? { + compiled += 1; + } + } + Ok((compiled, files.len(), started.elapsed().as_secs_f64() * 1e3)) } struct Module { @@ -449,12 +691,24 @@ struct Module { bytes: usize, load_ms: f64, from_cache: bool, - program: Program, + code: Code, /// Diagnostic high-water mark only: guest memory is never retained. memory_bytes: usize, invocations: u64, } +enum Code { + /// A core module (ADR 0011): a fresh instance for every call. + Module(Program), + /// A component (ADR 0014): one instance, kept between calls, and the + /// grants it was made with. + Component { + program: ComponentProgram, + live: Option<(ComponentInstance, Grants)>, + instances: u64, + }, +} + /// How one function has run: in the worker, from input to output. #[derive(Default)] struct Stats { @@ -466,9 +720,20 @@ struct Stats { impl Lab { fn load(app: &str, job: &Job) -> Result { - let admission = job.check(app)?; - let bundle = &admission.bundle; + let admission = job.check(app)?.clone(); let runtime = runtime(&job.host_dir)?; + Lab::from_bundle(app, runtime, admission, || job.check(app).map(drop)) + } + + /// The admitted bundle's `fns/*.wasm`, loaded; `check` ends it early (a + /// cancelled request, a changed admission). + fn from_bundle( + app: &str, + runtime: &'static Runtime, + admission: Admission, + check: impl Fn() -> Result<(), String>, + ) -> Result { + let bundle = &admission.bundle; let mut files: Vec = std::fs::read_dir(bundle.join("fns")) .map_err(|_| format!("{app}'s bundle has no fns directory"))? .flatten() @@ -490,10 +755,20 @@ impl Lab { modules: Vec::new(), owner: BTreeMap::new(), stats: BTreeMap::new(), + host_calls: Arc::new(AppHostCalls { + app: app.to_string(), + host_dir: admission.root.join(".host"), + families: admission.manifest["capabilities"] + .as_array() + .into_iter() + .flatten() + .filter_map(|c| c.as_str().map(str::to_string)) + .collect(), + }), admission: admission.clone(), }; for path in files { - job.check(app)?; + check()?; let file = path .file_name() .unwrap_or_default() @@ -506,9 +781,29 @@ impl Lab { } let bytes = std::fs::read(&path).map_err(|e| format!("{file}: {e}"))?; let started = Instant::now(); - let program = runtime.load(&bytes).map_err(|e| format!("{file}: {e}"))?; + let (code, functions, from_cache) = if component::is_component(&bytes) { + let program = runtime + .load_component(&bytes) + .map_err(|e| format!("{file}: {e}"))?; + let names: Vec = program.exports().iter().map(|e| e.name.clone()).collect(); + let from_cache = program.from_cache(); + ( + Code::Component { + program, + live: None, + instances: 0, + }, + names, + from_cache, + ) + } else { + let program = runtime.load(&bytes).map_err(|e| format!("{file}: {e}"))?; + let names = program.functions().to_vec(); + let from_cache = program.from_cache(); + (Code::Module(program), names, from_cache) + }; let load_ms = started.elapsed().as_secs_f64() * 1e3; - for function in program.functions() { + for function in &functions { if lab .owner .insert(function.clone(), lab.modules.len()) @@ -519,23 +814,27 @@ impl Lab { )); } } - let from_cache = program.from_cache(); makepad_widgets::log!( - "wasm {app}: {file} ({} KiB) {} in {load_ms:.1} ms: {}", + "wasm {app}: {file} ({} KiB{}) {} in {load_ms:.1} ms: {}", bytes.len() / 1024, + if matches!(code, Code::Component { .. }) { + ", a component" + } else { + "" + }, if from_cache { "loaded from the cache" } else { "compiled" }, - program.functions().join(", ") + functions.join(", ") ); lab.modules.push(Module { file, bytes: bytes.len(), load_ms, from_cache, - program, + code, memory_bytes: 0, invocations: 0, }); @@ -543,10 +842,18 @@ impl Lab { Ok(lab) } + /// Whether a component's instance is alive (its state worth keeping). + fn holds_instances(&self) -> bool { + self.modules + .iter() + .any(|m| matches!(&m.code, Code::Component { live: Some(_), .. })) + } + fn answer( &mut self, method: &str, input: &[u8], + text: bool, deadline: Instant, reply: Replier, ) -> Result { @@ -557,14 +864,25 @@ impl Lab { makepad_widgets::log!("wasm {}: {described}", self.app); return Ok(described); } - let Some(&index) = self.owner.get(method) else { + let name = if self.owner.contains_key(method) { + method.to_string() + } else { + component::snake(method) + }; + let Some(&index) = self.owner.get(&name) else { return Err(format!("{} has no function {method:?}", self.app)); }; let module = &mut self.modules[index]; + let program = match &mut module.code { + Code::Module(program) => program, + Code::Component { .. } => { + return self.answer_component(index, &name, input, text, deadline, reply) + } + }; let started = Instant::now(); let mut instance = self .runtime - .instantiate_guarded(&module.program, deadline, move || reply.is_pending()) + .instantiate_guarded(program, deadline, move || reply.is_pending()) .map_err(|error| error.to_string())?; module.invocations += 1; let result = instance.call(method, input); @@ -588,13 +906,101 @@ impl Lab { .unwrap_or_else(|_| json!({"text": String::from_utf8_lossy(&output)}))) } + /// One call to a component's function, on its live instance (made, or + /// remade, when there is none, it was spent, or its grants changed). + fn answer_component( + &mut self, + index: usize, + name: &str, + input: &[u8], + text: bool, + deadline: Instant, + reply: Replier, + ) -> Result { + let args: Value = if text { + Value::String(String::from_utf8_lossy(input).into_owned()) + } else if input.is_empty() { + Value::Null + } else { + serde_json::from_slice(input).map_err(|e| format!("the arguments are not JSON: {e}"))? + }; + // Decided for every call: a sign-out or a revoked grant applies to + // the next call, and the quota is what is left of it now. + let area = areas::app_area(&*area_env(), &self.app, true).ok(); + let grants = Grants { + storage_dir: area.as_ref().map(|a| a.root.clone()), + read_only: false, + }; + let app = self.app.clone(); + let host_calls = self.host_calls.clone(); + let module = &mut self.modules[index]; + let Code::Component { + program, + live, + instances, + } = &mut module.code + else { + unreachable!("answer_component is called for components"); + }; + let started = Instant::now(); + if live + .as_ref() + .is_some_and(|(instance, made)| instance.spent() || *made != grants) + { + *live = None; + } + if live.is_none() { + let pending = reply.clone(); + let instance = self + .runtime + .instantiate_component( + program, + &grants, + Some((deadline, Box::new(move || pending.is_pending()))), + ) + .map_err(|error| error.to_string())?; + let mut instance = instance; + instance.set_host_calls(Some(host_calls)); + *instances += 1; + *live = Some((instance, grants.clone())); + } + let (instance, _) = live.as_mut().expect("made above"); + module.invocations += 1; + // A write past the quota fails inside the component, as a full disk. + instance.set_storage_budget(area.and_then(|a| a.quota_left)); + let result = instance.call_json_guarded(name, &args, deadline, move || reply.is_pending()); + let us = started.elapsed().as_secs_f64() * 1e6; + for line in instance.take_logs() { + makepad_widgets::log!("wasm {app}: {line}"); + } + if instance.spent() { + *live = None; + } + let stats = self.stats.entry(name.to_string()).or_default(); + stats.calls += 1; + stats.total_us += us; + stats.max_us = stats.max_us.max(us); + result.map_err(|error| { + stats.errors += 1; + error.to_string() + }) + } + fn describe(&self) -> Value { let modules: Vec = self .modules .iter() - .map(|m| { - json!({"file": m.file, "bytes": m.bytes, "load_ms": round(m.load_ms), - "from_cache": m.from_cache, "memory_bytes": m.memory_bytes, "invocations": m.invocations, "renewed": m.invocations.saturating_sub(1), "instance_policy": "fresh-per-call"}) + .map(|m| match &m.code { + Code::Module(_) => json!({"file": m.file, "kind": "module", "bytes": m.bytes, "load_ms": round(m.load_ms), + "from_cache": m.from_cache, "memory_bytes": m.memory_bytes, "invocations": m.invocations, "renewed": m.invocations.saturating_sub(1), "instance_policy": "fresh-per-call"}), + Code::Component { program, live, instances } => json!({"file": m.file, "kind": "component", "bytes": m.bytes, "load_ms": round(m.load_ms), + "from_cache": m.from_cache, "invocations": m.invocations, "instances": instances, "instance_policy": "kept-between-calls", + "live": live.is_some(), "storage": live.as_ref().map(|(_, g)| if g.storage_dir.is_some() { "app folder" } else { "none" }), + "storage_left": live.as_ref().and_then(|(i, _)| i.storage_budget()), + "network": program.uses_network(), + "exports": program.exports().iter().map(|e| json!({"name": e.name, "wit": e.wit_name, + "params": e.params.iter().map(|(n, t)| json!([n, t])).collect::>(), "result": e.result})).collect::>(), + "skipped": program.skipped().iter().map(|(n, why)| json!({"name": n, "why": why})).collect::>()}), }) .collect(); let stats: serde_json::Map = self @@ -738,7 +1144,8 @@ mod tests { }); } - fn captured_job(app: &str, method: &str, host_dir: &Path) -> (usize, Job) { + /// A pending reply to `app`'s request, as dispatch hands a service one. + fn pending_reply(app: &str, host_dir: &Path) -> (usize, Replier) { struct Capture(Arc>>); impl HostService for Capture { fn family(&self) -> &'static str { @@ -765,11 +1172,17 @@ mod tests { &mut NoSheet, ); let reply = captured.lock().unwrap().take().unwrap(); + (heap, reply) + } + + fn captured_job(app: &str, method: &str, host_dir: &Path) -> (usize, Job) { + let (heap, reply) = pending_reply(app, host_dir); ( heap, Job { method: method.into(), input: Vec::new(), + text: false, host_dir: host_dir.into(), admission: Arc::new(AdmissionEpoch { admitted: OnceLock::from(Ok(Admission::current(app).unwrap())), @@ -1311,4 +1724,505 @@ mod tests { assert!(answer.unwrap_err().contains("no sheet")); let _ = std::fs::remove_dir_all(root); } + + /// The spike's component (ADR 0014): `pulldown-cmark` and friends, built + /// with plain cargo for `wasm32-wasip2`. + const NOTES_COMPONENT: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../wasm-host/tests/fixtures/notes.component.wasm" + ); + + /// A component's functions, in a process of its own (the area env and + /// the registries are process-wide): they answer with typed JSON under + /// either spelling of their names; one instance keeps its state between + /// calls until a trap spends it; and its files are the app's own + /// storage, under what is left of its quota, and absent without the + /// storage capability. + #[test] + fn a_components_instance_keeps_state_and_its_files_stay_in_the_apps_storage() { + const CHILD: &str = "OCTOSENSE_TEST_WASM_COMPONENT"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "wasm_service::tests::a_components_instance_keeps_state_and_its_files_stay_in_the_apps_storage", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + const APP: &str = "os.wasmnotes"; + const PLAIN: &str = "os.wasmplain"; + let root = + std::env::temp_dir().join(format!("octosense-wasm-component-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let bundle = root.join("bundle"); + std::fs::create_dir_all(bundle.join("fns")).unwrap(); + std::fs::copy(NOTES_COMPONENT, bundle.join("fns/notes.wasm")).unwrap(); + // The apps' storage as the shell keeps it: a 4 KiB ceiling for one, + // no storage capability for the other. + let storage = crate::app_storage::Storage::with_file_secrets( + crate::app_storage::Layout::new(&root.join("home")).unwrap(), + ); + let mut env = areas::FixedEnv { + storage: Some(storage.clone()), + ..areas::FixedEnv::default() + }; + env.quotas.insert( + APP.into(), + areas::JailQuota { + bytes: Some(4096), + storage: true, + }, + ); + env.quotas.insert( + PLAIN.into(), + areas::JailQuota { + bytes: None, + storage: false, + }, + ); + *AREA_ENV.lock().unwrap() = Some(Arc::new(env)); + let host_dir = root.join(".host"); + let runtime = runtime(&host_dir).unwrap(); + let admission = Admission { + root: root.clone(), + bundle, + manifest: json!({}), + }; + let mut lab = Lab::from_bundle(APP, runtime, admission.clone(), || Ok(())).unwrap(); + let (_, reply) = pending_reply(APP, &host_dir); + // What the app's script gets from `host.request("wasm." + method, args)`. + let call = |lab: &mut Lab, method: &str, args: Value| { + let text = matches!(args, Value::String(_)); + let input = input_bytes(&args).unwrap(); + lab.answer( + method, + &input, + text, + Instant::now() + REQUEST_TIMEOUT, + reply.clone(), + ) + }; + + // Typed JSON: a bare value for one parameter, an object by name or + // an array in order; a record comes back as an object. + assert_eq!( + call(&mut lab, "to_html", json!("# Hi")).unwrap(), + "

Hi

\n" + ); + assert_eq!( + call(&mut lab, "to-html", json!({"markdown": "*a*"})).unwrap(), + "

a

\n" + ); + assert_eq!( + call(&mut lab, "analyze", json!(["# One\n\ntwo words"])).unwrap(), + json!({"words": 4, "lines": 3, "headings": ["One"]}) + ); + let error = call(&mut lab, "nothing", json!({})).unwrap_err(); + assert!(error.contains("no function"), "{error}"); + + // One instance, kept between calls, until a trap spends it. + assert_eq!(call(&mut lab, "count", json!({})).unwrap(), 1); + assert_eq!(call(&mut lab, "count", Value::Null).unwrap(), 2); + let error = lab + .answer( + "spin", + b"{}", + false, + Instant::now() + Duration::from_millis(100), + reply.clone(), + ) + .unwrap_err(); + assert!(error.contains("deadline"), "{error}"); + assert_eq!(call(&mut lab, "count", json!({})).unwrap(), 1); + let described = lab.describe(); + let notes = &described["modules"][0]; + assert_eq!(notes["kind"], "component", "{described}"); + assert_eq!(notes["instance_policy"], "kept-between-calls"); + assert_eq!(notes["instances"], 2); + assert_eq!(notes["storage"], "app folder"); + assert!( + notes["exports"] + .as_array() + .unwrap() + .iter() + .any(|e| e["name"] == "save_html" && e["wit"] == "save-html"), + "{described}" + ); + assert!(described["functions"] + .as_array() + .unwrap() + .contains(&json!("echo_bytes"))); + assert_eq!(described["stats"]["spin"]["errors"], 1); + + // Its files are the app's own storage, the folder its script's fs.* + // sees, and nothing outside it. + let jail = storage.layout().app(APP).unwrap().jail; + let saved = json!({"markdown": "# Hi", "path": "a.html"}); + assert_eq!(call(&mut lab, "save_html", saved.clone()).unwrap(), 12); + assert_eq!( + std::fs::read_to_string(jail.join("a.html")).unwrap(), + "

Hi

\n" + ); + assert_eq!( + call(&mut lab, "read_file", json!("a.html")).unwrap(), + "

Hi

\n" + ); + std::fs::write(root.join("outside.txt"), "host").unwrap(); + for outside in ["../outside.txt", root.join("outside.txt").to_str().unwrap()] { + assert!( + call(&mut lab, "read_file", json!(outside)).is_err(), + "{outside}" + ); + } + + // The quota (4 KiB here): what is left of it is what a call may add. + // A write past it fails inside the component, which keeps its + // instance and state; a smaller one still goes in. + let before = call(&mut lab, "count", json!({})) + .unwrap() + .as_u64() + .unwrap(); + let big = json!({"markdown": "x".repeat(5000), "path": "big.html"}); + let error = call(&mut lab, "save_html", big).unwrap_err(); + assert!(error.ends_with("the storage budget is used up"), "{error}"); + assert_eq!(std::fs::metadata(jail.join("big.html")).unwrap().len(), 0); + let small = json!({"markdown": "# Hi", "path": "b.html"}); + assert_eq!(call(&mut lab, "save_html", small).unwrap(), 12); + // What the app writes otherwise counts too, from the next call on. + std::fs::write(jail.join("filler"), vec![0u8; 4096 - 24]).unwrap(); + let error = call( + &mut lab, + "save_html", + json!({"markdown": "# Hi", "path": "c.html"}), + ) + .unwrap_err(); + assert!(error.ends_with("the storage budget is used up"), "{error}"); + assert_eq!(call(&mut lab, "count", json!({})).unwrap(), before + 1); + let described = lab.describe(); + assert_eq!(described["modules"][0]["storage"], "app folder"); + assert_eq!(described["modules"][0]["storage_left"], 0); + assert_eq!(described["modules"][0]["instances"], 2); + + // Without the storage capability, no folder at all. + let mut plain = Lab::from_bundle(PLAIN, runtime, admission, || Ok(())).unwrap(); + let (_, plain_reply) = pending_reply(PLAIN, &host_dir); + let error = plain + .answer( + "save_html", + saved.to_string().as_bytes(), + false, + Instant::now() + REQUEST_TIMEOUT, + plain_reply.clone(), + ) + .unwrap_err(); + assert!(!error.is_empty()); + assert_eq!(plain.describe()["modules"][0]["storage"], "none"); + let _ = std::fs::remove_dir_all(root); + } + + /// A component through the whole path, in a process of its own: shipped + /// as a system app that requires `wasm-components-v1` (App Hub's + /// admission reads the component's imports), its functions answer the + /// app's script with typed JSON, its instance keeps its state from one + /// request to the next, and without `storage` it has no folder. + #[test] + fn a_shipped_component_answers_its_apps_script() { + const CHILD: &str = "OCTOSENSE_TEST_WASM_SHIPPED_COMPONENT"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "wasm_service::tests::a_shipped_component_answers_its_apps_script", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let root = std::env::temp_dir().join(format!( + "octosense-wasm-shipped-component-{}", + std::process::id() + )); + let host_dir = root.join(".host"); + // This exact test runs alone in its child process. Keep the env + // override and registered root identical even under an isolated suite. + std::env::set_var("OCTOSENSE_APP_DATA", &root); + octosense_appstore::set_data_root(root.clone()); + ship("os.wasmnotes", "component", |dir, manifest| { + manifest["capabilities"] = json!(["wasm"]); + manifest["requires"] = json!(["wasm-components-v1"]); + manifest.as_object_mut().unwrap().remove("agent"); + std::fs::remove_file(dir.join("tools.json")).unwrap(); + std::fs::remove_file(dir.join("AGENT.md")).unwrap(); + std::fs::remove_file(dir.join("fns/wasmlab.wasm")).unwrap(); + std::fs::copy(NOTES_COMPONENT, dir.join("fns/notes.wasm")).unwrap(); + }); + register(); + let call = |method: &str, args: Value| request("os.wasmnotes", method, args, &host_dir); + + assert_eq!( + call("wasm.to_html", json!("# Hi")).unwrap(), + "

Hi

\n" + ); + assert_eq!( + call("wasm.analyze", json!({"markdown": "# One\n\ntwo words"})).unwrap(), + json!({"words": 4, "lines": 3, "headings": ["One"]}) + ); + assert_eq!(call("wasm.count", json!({})).unwrap(), 1); + assert_eq!(call("wasm.count", json!({})).unwrap(), 2); + let error = call( + "wasm.save_html", + json!({"markdown": "# Hi", "path": "a.html"}), + ) + .unwrap_err(); + assert!(!error.is_empty()); + let described = call("wasm.functions", json!({})).unwrap(); + let notes = &described["modules"][0]; + assert_eq!(notes["file"], "notes.wasm", "{described}"); + assert_eq!(notes["kind"], "component"); + assert_eq!(notes["instances"], 1); + assert_eq!(notes["storage"], "none"); + assert_eq!(described["stats"]["count"]["calls"], 2); + let _ = std::fs::remove_dir_all(root); + } + + /// A component's requests reach any host: an app's network + /// declarations are shown at install, not enforced (the ruling of + /// 8 October 2026), so an app with neither `net` nor `network.hosts` + /// reaches a server on this device. + #[test] + fn a_components_requests_reach_any_host() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + for mut stream in listener.incoming().flatten() { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if std::io::Read::read(&mut stream, &mut byte).unwrap_or(0) == 0 { + break; + } + request.push(byte[0]); + } + let _ = std::io::Write::write_all( + &mut stream, + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello", + ); + } + }); + let root = std::env::temp_dir().join(format!( + "octosense-wasm-component-net-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&root); + let bundle = root.join("bundle"); + std::fs::create_dir_all(bundle.join("fns")).unwrap(); + std::fs::copy( + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../wasm-host/tests/fixtures/fetch.component.wasm" + ), + bundle.join("fns/fetch.wasm"), + ) + .unwrap(); + let dir = script_apps::tests::stamped_bundle("wasmlab", "net", |_, _| {}); + let mut manifest: Value = + serde_json::from_str(&std::fs::read_to_string(dir.join("manifest.json")).unwrap()) + .unwrap(); + let _ = std::fs::remove_dir_all(&dir); + manifest["requires"] = json!(["wasm-components-v1"]); + manifest["capabilities"] = json!(["wasm"]); + assert!(manifest.get("network").is_none()); + let host_dir = root.join(".host"); + let runtime = runtime(&host_dir).unwrap(); + let admission = Admission { + root: root.clone(), + bundle: bundle.clone(), + manifest, + }; + let mut lab = Lab::from_bundle("os.wasmnet", runtime, admission, || Ok(())).unwrap(); + for host in ["127.0.0.1", "localhost"] { + let (_, reply) = pending_reply("os.wasmnet", &host_dir); + let url = json!(format!("http://{host}:{port}/hi")); + let answer = lab.answer( + "get", + url.to_string().as_bytes(), + false, + Instant::now() + REQUEST_TIMEOUT, + reply, + ); + assert_eq!(answer.unwrap(), "200 hello", "{host}"); + } + assert_eq!(lab.describe()["modules"][0]["network"], json!(true)); + let _ = std::fs::remove_dir_all(root); + } + + /// A component's `octosense:host` calls reach its app's granted host + /// services, dispatched on the UI thread as its script's would be (a + /// thread here plays the UI's part); nothing else, and never `wasm.*`. + #[test] + fn a_components_host_calls_reach_only_its_apps_granted_services() { + struct Echo; + impl HostService for Echo { + fn family(&self) -> &'static str { + "wasmhostecho" + } + fn call(&mut self, call: ServiceCall, reply: Replier, _: &mut dyn ServiceHost) { + reply.send(Ok(json!({"app": call.app_id, "method": call.method(), "args": call.args, "may_prompt": call.may_prompt}))); + } + } + octosense_appstore::services::register_host_service(Box::new(Echo)); + let stop = Arc::new(AtomicBool::new(false)); + let pumping = stop.clone(); + let pump = std::thread::spawn(move || { + while !pumping.load(Ordering::Relaxed) { + pump_host_calls(); + std::thread::sleep(Duration::from_millis(2)); + } + }); + let root = std::env::temp_dir().join(format!( + "octosense-wasm-component-host-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&root); + let bundle = root.join("bundle"); + std::fs::create_dir_all(bundle.join("fns")).unwrap(); + std::fs::copy( + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../wasm-host/tests/fixtures/hostcall.component.wasm" + ), + bundle.join("fns/hostcall.wasm"), + ) + .unwrap(); + let host_dir = root.join(".host"); + let admission = Admission { + root: root.clone(), + bundle, + manifest: json!({"capabilities": ["wasm", "wasmhostecho"]}), + }; + let mut lab = Lab::from_bundle( + "org.example.hostcalls", + runtime(&host_dir).unwrap(), + admission, + || Ok(()), + ) + .unwrap(); + let mut call = |service: &str, args: &str| { + let (_, reply) = pending_reply("org.example.hostcalls", &host_dir); + lab.answer( + "call", + json!([service, args]).to_string().as_bytes(), + false, + Instant::now() + REQUEST_TIMEOUT, + reply, + ) + }; + let answer: Value = serde_json::from_str( + call("wasmhostecho.get", r#"{"id":1}"#) + .unwrap() + .as_str() + .unwrap(), + ) + .unwrap(); + assert_eq!( + answer, + json!({"app": "org.example.hostcalls", "method": "get", "args": {"id": 1}, "may_prompt": false}) + ); + let error = call("mail.list", "{}").unwrap_err(); + assert!( + error.contains("was not granted the mail service"), + "{error}" + ); + let error = call("wasm.functions", "{}").unwrap_err(); + assert!(error.contains("cannot call wasm.*"), "{error}"); + let error = call("wasmhostecho.get", "not json").unwrap_err(); + assert!(error.contains("not JSON"), "{error}"); + stop.store(true, Ordering::Relaxed); + pump.join().unwrap(); + let _ = std::fs::remove_dir_all(root); + } + + /// An installed app's functions are compiled before its first call, in + /// a process of its own (the apps root and the registries are + /// process-wide): the first call loads them from the cache. + #[test] + fn an_installed_apps_first_call_loads_from_the_cache() { + const CHILD: &str = "OCTOSENSE_TEST_WASM_WARM"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "wasm_service::tests::an_installed_apps_first_call_loads_from_the_cache", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let root = std::env::temp_dir().join(format!("octosense-wasm-warm-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let host_dir = root.join(".host"); + // This exact test runs alone in its child process. Keep the env + // override and registered root identical even under an isolated suite. + std::env::set_var("OCTOSENSE_APP_DATA", &root); + octosense_appstore::set_data_root(root.clone()); + ship("os.wasmlab", "warm", |_, _| {}); + register(); + warm("os.wasmlab"); + let cache = host_dir.join("wasm-cache"); + let cached = || { + std::fs::read_dir(&cache) + .map(|entries| { + entries + .flatten() + .any(|e| e.path().extension().is_some_and(|x| x == "cwasm")) + }) + .unwrap_or(false) + }; + for _ in 0..3000 { + if cached() { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!(cached(), "warm compiled nothing into {}", cache.display()); + let ranked = request( + "os.wasmlab", + "wasm.fuzzy_rank", + json!({"query": "m", "items": ["Mail"]}), + &host_dir, + ) + .unwrap(); + assert_eq!(ranked["ranked"][0]["item"], "Mail"); + let described = request("os.wasmlab", "wasm.functions", json!({}), &host_dir).unwrap(); + assert_eq!(described["modules"][0]["from_cache"], true, "{described}"); + // Warming again finds it compiled. + assert_eq!(warm_now("os.wasmlab").unwrap().0, 0); + let _ = std::fs::remove_dir_all(root); + } } diff --git a/crates/wasm-host/Cargo.toml b/crates/wasm-host/Cargo.toml index 3db69b248..e012f253b 100644 --- a/crates/wasm-host/Cargo.toml +++ b/crates/wasm-host/Cargo.toml @@ -8,8 +8,14 @@ publish = false [dependencies] wasmtime = { workspace = true } wasmtime-wasi = { workspace = true } +wasmtime-wasi-http = { workspace = true } +# The requests a component's `wasi:http` sends, checked against its hosts. +http = "1" +http-body-util = "0.1" serde_json = { workspace = true } base64 = "0.22" +# File writes a component makes (wasmtime-wasi's streams carry `Bytes`). +bytes = "1" sha2 = "0.10" # Not used directly: Wasmtime reads a trap's registers through libc's # ucontext_t, and before 0.2.190 Android aarch64's lacked the kernel's @@ -17,6 +23,16 @@ sha2 = "0.10" # recognised as wasm and every trap ended the app. libc = "0.2.190" +# Pulley, Wasmtime's interpreter (ADR 0014 phase 3): how code runs on iOS, +# where an app may not generate code, and on OpenHarmony, until its policy is +# known. Elsewhere only for its tests (OCTOSENSE_WASM_PULLEY with this +# feature); desktop and Android builds leave it out. +[features] +pulley = ["wasmtime/pulley"] + +[target.'cfg(any(target_os = "ios", target_env = "ohos"))'.dependencies] +wasmtime = { workspace = true, features = ["pulley"] } + [dev-dependencies] wat = "1" # The sample functions natively, to check the module against them. diff --git a/crates/wasm-host/examples/measure_component.rs b/crates/wasm-host/examples/measure_component.rs new file mode 100644 index 000000000..f2e76e72e --- /dev/null +++ b/crates/wasm-host/examples/measure_component.rs @@ -0,0 +1,242 @@ +//! A component's (ADR 0014) startup and per-call latency in the runtime, +//! against Wasm Lab's core module (ADR 0011) and native code doing the same +//! work, and the cost of the storage quota check the shell's `wasm` service +//! makes after a component's call. +//! +//! cargo run --release -p octosense-wasm-host --example measure_component + +use std::path::Path; +use std::time::{Duration, Instant}; + +use octosense_wasm_host::component::Grants; +use octosense_wasm_host::{Limits, Runtime}; +use serde_json::{json, Value}; + +const COMPONENT: &[u8] = include_bytes!("../tests/fixtures/notes.component.wasm"); +const MODULE: &[u8] = include_bytes!("../../../apps/wasmlab/bundle/fns/wasmlab.wasm"); + +/// Median of `runs` timings of `f`, in microseconds. +fn median_us(runs: usize, mut f: impl FnMut()) -> f64 { + let mut times: Vec = (0..runs) + .map(|_| { + let t = Instant::now(); + f(); + t.elapsed() + }) + .collect(); + times.sort(); + times[runs / 2].as_secs_f64() * 1e6 +} + +fn markdown() -> String { + let mut text = String::new(); + for i in 0..200 { + text.push_str(&format!( + "## Section {i}\n\nSome *emphasis*, **strong** text, `code` and a [link](https://example.org/{i}).\n\n- [x] done {i}\n- [ ] todo {i}\n\n| key | value |\n|---|---|\n| a{i} | {i} |\n\n" + )); + } + text +} + +/// The walk the shell's `Storage::usage` makes over an app's jail: every +/// entry's metadata, never following a symlink. +fn tree_bytes(dir: &Path) -> u64 { + let mut total = 0; + let mut stack = vec![dir.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(meta) = std::fs::symlink_metadata(entry.path()) else { + continue; + }; + if meta.is_dir() { + stack.push(entry.path()); + } else if meta.is_file() { + total += meta.len(); + } + } + } + total +} + +fn main() { + println!( + "component: {} KiB (module: {} KiB)", + COMPONENT.len() / 1024, + MODULE.len() / 1024 + ); + let scratch = std::env::temp_dir().join(format!( + "octosense-wasm-measure-component-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&scratch); + let cache = scratch.join("cache"); + + let t = Instant::now(); + let runtime = Runtime::new(Limits::default(), Some(cache.clone())).unwrap(); + let program = runtime.load_component(COMPONENT).unwrap(); + println!( + "compile (Cranelift, first load): {:.1} ms", + t.elapsed().as_secs_f64() * 1e3 + ); + let again = Runtime::new(Limits::default(), Some(cache.clone())).unwrap(); + let t = Instant::now(); + let cached = again.load_component(COMPONENT).unwrap(); + println!( + "load from the cache: {:.2} ms (from cache: {})", + t.elapsed().as_secs_f64() * 1e3, + cached.from_cache() + ); + let bare = median_us(50, || { + drop( + runtime + .instantiate_component(&program, &Grants::default(), None) + .unwrap(), + ) + }); + let storage = scratch.join("jail"); + std::fs::create_dir_all(&storage).unwrap(); + let grants = Grants { + storage_dir: Some(storage.clone()), + read_only: false, + }; + let with_files = median_us(50, || { + drop( + runtime + .instantiate_component(&program, &grants, None) + .unwrap(), + ) + }); + println!( + "instantiate: {:.3} ms; with the storage folder: {:.3} ms", + bare / 1e3, + with_files / 1e3 + ); + + let mut component = runtime + .instantiate_component(&program, &grants, None) + .unwrap(); + let module = runtime.load(MODULE).unwrap(); + let mut guest = runtime.instantiate(&module).unwrap(); + let md = markdown(); + let md_json = json!({"markdown": md}); + println!( + "\n{:<46} {:>10} {:>10} {:>10}", + "per call (median of 200)", "component", "module", "native us" + ); + let to_html = median_us(200, || { + drop(component.call_json("to_html", &md_json).unwrap()) + }); + let md_to_html = median_us(200, || { + drop(guest.call("md_to_html", md.as_bytes()).unwrap()) + }); + let native = median_us(200, || { + drop(wasmlab_functions::md_to_html(md.as_bytes()).unwrap()) + }); + println!( + "{:<46} {to_html:>10.1} {md_to_html:>10.1} {native:>10.1}", + format!("Markdown to HTML ({} KiB, pulldown-cmark)", md.len() / 1024) + ); + let analyze = median_us(200, || { + drop(component.call_json("analyze", &md_json).unwrap()) + }); + println!( + "{:<46} {analyze:>10.1} {:>10} {:>10}", + "analyze (a record back)", "-", "-" + ); + // A new file every time: rewriting one file over and over measures the + // filesystem (APFS takes about 6 ms a rewrite, natively too). + let mut n = 0; + let mut saved = || { + n += 1; + json!({"markdown": md, "path": format!("out{n}.html")}) + }; + let save = median_us(200, || { + drop(component.call_json("save_html", &saved()).unwrap()) + }); + println!( + "{:<46} {save:>10.1} {:>10} {:>10}", + "save_html (render, write 49 KiB)", "-", "-" + ); + let html = component.call_json("to_html", &md_json).unwrap(); + let html = html.as_str().unwrap().to_string(); + let mut m = 0; + let native_save = median_us(200, || { + m += 1; + std::fs::write(storage.join(format!("native{m}.html")), &html).unwrap() + }); + println!( + "{:<46} {:>10} {:>10} {native_save:>10.1}", + "write 49 KiB natively", "-", "-" + ); + let mut k = 0; + let save_tiny = median_us(200, || { + k += 1; + drop( + component + .call_json( + "save_html", + &json!({"markdown": "# Hi", "path": format!("tiny{k}.html")}), + ) + .unwrap(), + ) + }); + println!( + "{:<46} {save_tiny:>10.1} {:>10} {:>10}", + "save_html (write 12 bytes)", "-", "-" + ); + let read = json!("out1.html"); + let read_file = median_us(200, || { + drop(component.call_json("read_file", &read).unwrap()) + }); + println!( + "{:<46} {read_file:>10.1} {:>10} {:>10}", + "read_file (49 KiB back)", "-", "-" + ); + let mib: Value = json!(base64_of(&vec![7u8; 1 << 20])); + let echo = median_us(50, || { + drop(component.call_json("echo_bytes", &mib).unwrap()) + }); + println!( + "{:<46} {echo:>10.1} {:>10} {:>10}", + "echo_bytes (1 MiB, base64 both ways)", "-", "-" + ); + let empty = json!({"query": "x", "items": [], "limit": 1}); + let smallest_module = median_us(2000, || { + drop( + guest + .call("fuzzy_rank", &serde_json::to_vec(&empty).unwrap()) + .unwrap(), + ) + }); + let smallest = median_us(2000, || { + drop(component.call_json("count", &Value::Null).unwrap()) + }); + println!( + "\nsmallest call: component `count` {smallest:.1} us; module `fuzzy_rank` over no items {smallest_module:.1} us" + ); + + // The quota check after a call that had the storage folder: one walk of + // the app's jail, as `Storage::usage` makes it. + println!("\nquota check (one walk of the jail, median of 50):"); + for files in [10usize, 1_000, 10_000] { + let jail = scratch.join(format!("jail-{files}")); + for i in 0..files { + let dir = jail.join(format!("d{}", i / 100)); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("f{i}")), b"x").unwrap(); + } + let walk = median_us(50, || { + std::hint::black_box(tree_bytes(&jail)); + }); + println!(" {files:>6} files: {:.2} ms", walk / 1e3); + } + let _ = std::fs::remove_dir_all(&scratch); +} + +fn base64_of(bytes: &[u8]) -> String { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.encode(bytes) +} diff --git a/crates/wasm-host/src/component.rs b/crates/wasm-host/src/component.rs index 98e4261a5..a4c21f6e7 100644 --- a/crates/wasm-host/src/component.rs +++ b/crates/wasm-host/src/component.rs @@ -23,6 +23,11 @@ //! options are `null` or the value, `list` is base64 text, and a //! function's `result` is its value or its error. //! +//! Names face a script in snake_case: WIT's `to-html` is called as +//! `to_html`, and a record's `word-count` comes back as `word_count`, so a +//! script reads `r.data.word_count`. Either spelling is accepted on the +//! way in. +//! //! Unlike a core module's, an instance keeps its state between calls: a //! component may hold a parsed document, a cache or a model. A trap or a //! deadline spends it, as with modules. @@ -30,7 +35,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use base64::Engine as _; use serde_json::{json, Map, Value as Json}; @@ -40,9 +45,13 @@ use wasmtime::{Store, StoreLimits, StoreLimitsBuilder}; use wasmtime_wasi::p2::pipe::MemoryOutputPipe; use wasmtime_wasi::{FsPerms, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; -use crate::{ - trap, CallError, InvocationGuard, LoadError, Runtime, Ticker, LOG_LINE, LOG_LINES, TICK, -}; +use crate::{trap, CallError, InvocationGuard, LoadError, Runtime, Ticker, LOG_LINE, LOG_LINES}; + +mod files; +mod host; +mod net; + +pub use host::HostCalls; /// The WASI packages a component may import. Every interface of these is /// linked; what each can reach is set per instance ([`Grants`]). @@ -50,10 +59,16 @@ const ALLOWED_PACKAGES: &[&str] = &[ "wasi:cli/", "wasi:clocks/", "wasi:filesystem/", + "wasi:http/", "wasi:io/", "wasi:random/", + // The app's host services, as its script reaches them (`host`). + "octosense:host/", ]; +/// What a call that had a write refused logs, and adds to its error. +const REFUSED: &str = "a write was refused: the storage budget is used up"; + /// How much one stream (stdout or stderr) buffers between reads. const PIPE_BYTES: usize = 64 << 10; @@ -64,18 +79,22 @@ pub fn is_component(bytes: &[u8]) -> bool { } /// What an instance of a component may reach beyond clocks and randomness. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Grants { /// The app's storage folder, preopened as `/`. `None`: no filesystem. pub storage_dir: Option, + /// Open the storage folder read-only (its quota is used up). + pub read_only: bool, } /// One function an app may call, with its WIT-like signature. #[derive(Clone, Debug, PartialEq)] pub struct Export { - /// `name` for a world's own function, `interface.name` for one in an - /// exported interface. + /// The name a script calls it by, in snake_case: `name` for a world's + /// own function, `interface.name` for one in an exported interface. pub name: String, + /// The same as WIT spells it (`to-html`, `markdown.to-html`). + pub wit_name: String, /// Parameter names and their types, as WIT writes them. pub params: Vec<(String, String)>, /// The result's type, as WIT writes it, if any. @@ -91,6 +110,8 @@ pub struct ComponentProgram { /// the reason, so `wasm.functions` can say why they are missing. skipped: Vec<(String, String)>, from_cache: bool, + /// It imports `wasi:http`, so its calls may wait for the network. + uses_network: bool, } impl ComponentProgram { @@ -108,12 +129,28 @@ impl ComponentProgram { pub fn from_cache(&self) -> bool { self.from_cache } + + /// Whether it imports `wasi:http`: it may reach any host, and its calls + /// get the network deadline. + pub fn uses_network(&self) -> bool { + self.uses_network + } } struct State { wasi: WasiCtx, table: ResourceTable, limits: StoreLimits, + /// The running call's deadline and cancellation, read every epoch. + guard: Option>, + /// What the running call may add to the storage folder. + budget: files::Budget, + http: wasmtime_wasi_http::WasiHttpCtx, + /// The running call's deadline, for the requests it sends. + network: net::Network, + /// Its app's host services (`octosense:host`), when the embedder gives + /// them. + host_calls: host::Calls, } impl WasiView for State { @@ -125,6 +162,16 @@ impl WasiView for State { } } +impl wasmtime_wasi_http::WasiHttpView for State { + fn http(&mut self) -> wasmtime_wasi_http::WasiHttpCtxView<'_> { + wasmtime_wasi_http::WasiHttpCtxView { + ctx: &mut self.http, + table: &mut self.table, + hooks: &mut self.network, + } + } +} + /// One app's instance of a [`ComponentProgram`]. It keeps its state between /// calls until a trap or a deadline spends it. pub struct ComponentInstance { @@ -135,10 +182,9 @@ pub struct ComponentInstance { /// Bytes of stdout and stderr already turned into log lines. read: [usize; 2], logs: Vec, - deadline_ticks: u64, + deadline: Duration, io_bytes: usize, spent: bool, - guard: Option>, _ticker: Arc, } @@ -165,11 +211,16 @@ impl Runtime { None => self.compile_component(bytes, cached.as_ref())?, }; let (exports, skipped) = check_component(&self.engine, &component)?; + let uses_network = component + .component_type() + .imports(&self.engine) + .any(|(name, _)| name.starts_with("wasi:http/")); Ok(ComponentProgram { component, exports, skipped, from_cache, + uses_network, }) } @@ -212,57 +263,91 @@ impl Runtime { .stderr(stderr.clone()) .allow_tcp(false) .allow_udp(false) - .allow_ip_name_lookup(false); + .allow_ip_name_lookup(false) + // Calls run on their app's own worker thread, synchronously: a + // file operation blocks it directly rather than round-tripping + // through a thread pool. + .allow_blocking_current_thread(true); if let Some(dir) = &grants.storage_dir { - wasi.preopened_dir(dir, "/", FsPerms::ReadWrite) - .map_err(|e| LoadError::Invalid(format!("the app's storage: {e:#}")))?; + wasi.preopened_dir( + dir, + "/", + if grants.read_only { + FsPerms::ReadOnly + } else { + FsPerms::ReadWrite + }, + ) + .map_err(|e| LoadError::Invalid(format!("the app's storage: {e:#}")))?; } let limits = StoreLimitsBuilder::new() .memory_size(self.limits.memory_bytes) .table_elements(self.limits.table_elements) .trap_on_grow_failure(true) .build(); + // Instantiation runs the component's start-up code: it gets the + // same deadline as a call, and the caller's cancellation if any. + let (deadline, pending) = match guard { + Some((deadline, pending)) => (deadline, pending), + None => ( + Instant::now() + self.limits.deadline, + Box::new(|| true) as Box bool + Send + Sync>, + ), + }; + // A component that imports wasi:http waits for replies: its calls + // get the longer deadline. + let call_deadline = if program.uses_network { + self.limits.network_deadline.max(self.limits.deadline) + } else { + self.limits.deadline + }; + let guard = Arc::new(InvocationGuard { + deadline: deadline.min(Instant::now() + call_deadline), + pending, + }); + let network = net::Network { + deadline: Some(guard.deadline), + }; let mut store = Store::new( &self.engine, State { wasi: wasi.build(), table: ResourceTable::new(), limits, + guard: Some(guard), + // Start-up code may read the folder, not grow it. + budget: files::Budget::default(), + http: wasmtime_wasi_http::WasiHttpCtx::new(), + network, + host_calls: None, }, ); + store.data().budget.set(Some(0)); store.limiter(|state| &mut state.limits); - let guard = guard.map(|(deadline, pending)| { - let deadline = deadline.min(Instant::now() + self.limits.deadline); - Arc::new(InvocationGuard { deadline, pending }) + // Every epoch, the running call's guard decides: a call never runs + // without one, so none outlives its deadline. + store.epoch_deadline_callback(|ctx| { + Ok(match &ctx.data().guard { + Some(guard) if guard.live() => wasmtime::UpdateDeadline::Continue(1), + _ => wasmtime::UpdateDeadline::Interrupt, + }) }); - if let Some(guard) = guard.clone() { - store.epoch_deadline_callback(move |_| { - Ok(if !guard.live() { - wasmtime::UpdateDeadline::Interrupt - } else { - wasmtime::UpdateDeadline::Continue(1) - }) - }); - } else { - store.epoch_deadline_trap(); - } - let deadline_ticks = self - .limits - .deadline - .as_millis() - .div_ceil(TICK.as_millis()) - .max(1) as u64; - store.set_epoch_deadline(if guard.is_some() { 1 } else { deadline_ticks }); + store.set_epoch_deadline(1); let mut linker = Linker::new(&self.engine); wasmtime_wasi::p2::add_to_linker_sync(&mut linker) + .and_then(|()| wasmtime_wasi_http::p2::add_only_http_to_linker_sync(&mut linker)) + .and_then(|()| files::link(&mut linker)) + .and_then(|()| host::link(&mut linker)) .map_err(|e| LoadError::Invalid(format!("{e:#}")))?; let instance = linker .instantiate(&mut store, &program.component) .map_err(|e| LoadError::Invalid(format!("{e:#}")))?; + store.data_mut().guard = None; + store.data().budget.set(None); let mut funcs = HashMap::new(); for export in &program.exports { - let func = match export.name.split_once('.') { - None => instance.get_func(&mut store, &export.name), + let func = match export.wit_name.split_once('.') { + None => instance.get_func(&mut store, &export.wit_name), Some((iface, name)) => { let full = interface_export(&program.component, &self.engine, iface); full.and_then(|full| instance.get_export_index(&mut store, None, &full)) @@ -280,10 +365,9 @@ impl Runtime { stderr, read: [0, 0], logs: Vec::new(), - deadline_ticks, + deadline: call_deadline, io_bytes: self.limits.io_bytes, spent: false, - guard, _ticker: self.ticker.clone(), }) } @@ -305,6 +389,11 @@ fn interface_export( .map(|(name, _)| name.to_string()) } +/// WIT's kebab-case as a script spells it: `word-count` → `word_count`. +pub fn snake(name: &str) -> String { + name.replace('-', "_") +} + /// `my:pkg/markdown@0.1.0` → `markdown`. fn short_name(full: &str) -> &str { let tail = full.rsplit('/').next().unwrap_or(full); @@ -367,8 +456,9 @@ fn add_export( return; } exports.push(Export { - name, - params: params.iter().map(|(n, t)| (n.clone(), wit(t))).collect(), + name: snake(&name), + wit_name: name, + params: params.iter().map(|(n, t)| (snake(n), wit(t))).collect(), result: match results.len() { 0 => None, 1 => Some(wit(&results[0])), @@ -486,19 +576,53 @@ impl ComponentInstance { self.spent } + /// What the next calls may add to the storage folder (`None`, the + /// default: no ceiling). A write past it fails in the guest as a full + /// disk; overwriting, truncating and deleting give bytes back. + pub fn set_storage_budget(&mut self, left: Option) { + self.store.data().budget.set(left); + } + + /// What is left of the storage budget. + pub fn storage_budget(&self) -> Option { + self.store.data().budget.left() + } + + /// The app's host services its `octosense:host` calls reach (`None`, + /// the default: none). + pub fn set_host_calls(&mut self, calls: Option>) { + self.store.data_mut().host_calls = calls; + } + /// Calls `name` with JSON arguments; returns its JSON result. A /// `result` return is `T`, or [`CallError::Guest`] with `E`. pub fn call_json(&mut self, name: &str, args: &Json) -> Result { + self.call_json_guarded(name, args, Instant::now() + self.deadline, || true) + } + + /// [`ComponentInstance::call_json`], ended at `deadline` (no later than + /// the runtime's) or as soon as `pending` answers false. + pub fn call_json_guarded( + &mut self, + name: &str, + args: &Json, + deadline: Instant, + pending: impl Fn() -> bool + Send + Sync + 'static, + ) -> Result { if self.spent { return Err(CallError::Spent); } - if self.guard.as_ref().is_some_and(|guard| !guard.live()) { - self.spent = true; + let guard = Arc::new(InvocationGuard { + deadline: deadline.min(Instant::now() + self.deadline), + pending: Box::new(pending), + }); + if !guard.live() { return Err(CallError::Deadline); } let func = *self .funcs .get(name) + .or_else(|| self.funcs.get(&snake(name))) .ok_or_else(|| CallError::NoSuchFunction(name.into()))?; let size = serde_json::to_vec(args).map(|v| v.len()).unwrap_or(0); if size > self.io_bytes { @@ -509,15 +633,20 @@ impl ComponentInstance { let args = arguments(¶ms, args).map_err(CallError::Guest)?; let results_ty: Vec = fty.results().collect(); let mut results = vec![Val::Bool(false); results_ty.len()]; - self.store.set_epoch_deadline(if self.guard.is_some() { - 1 - } else { - self.deadline_ticks - }); + self.store.data_mut().network.deadline = Some(guard.deadline); + self.store.data_mut().guard = Some(guard); + self.store.set_epoch_deadline(1); let outcome = func .call(&mut self.store, &args, &mut results) .map_err(trap); + self.store.data_mut().guard = None; self.collect_logs(); + // A refused write reaches the guest as a full disk or, through a + // stream, a bare I/O error: say what it was. + let refused = self.store.data().budget.take_refused(); + if refused { + self.logs.push(REFUSED.into()); + } if let Err(error) = outcome { if matches!(error, CallError::Deadline | CallError::Trap(_)) { self.spent = true; @@ -532,13 +661,16 @@ impl ComponentInstance { .map(|(v, t)| from_val(&t, v)) .unwrap_or(Json::Null), ([Type::Result(res)], [Val::Result(Err(err))]) => { - return Err(CallError::Guest( - match err.as_deref().zip(res.err()).map(|(v, t)| from_val(&t, v)) { - Some(Json::String(text)) => text, - Some(other) => other.to_string(), - None => "the function failed".into(), - }, - )) + let why = match err.as_deref().zip(res.err()).map(|(v, t)| from_val(&t, v)) { + Some(Json::String(text)) => text, + Some(other) => other.to_string(), + None => "the function failed".into(), + }; + return Err(CallError::Guest(if refused { + format!("{why}; {REFUSED}") + } else { + why + })); } ([ty], [one]) => from_val(ty, one), (types, many) => Json::Array( @@ -588,36 +720,50 @@ fn arguments(params: &[(String, Type)], args: &Json) -> Result, String> let named = |map: &Map| -> Result, String> { params .iter() - .map(|(n, t)| to_val(t, &map[n]).map_err(|e| format!("{n}: {e}"))) + .map(|(n, t)| { + let value = field(map, n).unwrap_or(&Json::Null); + to_val(t, value).map_err(|e| format!("{}: {e}", snake(n))) + }) .collect() }; + let has_all = |map: &Map| { + map.len() == params.len() && params.iter().all(|(n, _)| field(map, n).is_some()) + }; match (params, args) { ([], Json::Null) => Ok(Vec::new()), ([], Json::Object(map)) if map.is_empty() => Ok(Vec::new()), ([], Json::Array(items)) if items.is_empty() => Ok(Vec::new()), - ([(name, _)], Json::Object(map)) if map.len() == 1 && map.contains_key(name) => named(map), - ([(name, ty)], value) => Ok(vec![to_val(ty, value).map_err(|e| format!("{name}: {e}"))?]), - (_, Json::Object(map)) - if map.len() == params.len() && params.iter().all(|(n, _)| map.contains_key(n)) => - { - named(map) - } + ([_], Json::Object(map)) if has_all(map) => named(map), + // The value itself or, when it is not one, a one-item array of it. + ([(name, ty)], value) => match (to_val(ty, value), value) { + (Ok(one), _) => Ok(vec![one]), + (Err(_), Json::Array(items)) if items.len() == 1 => Ok(vec![ + to_val(ty, &items[0]).map_err(|e| format!("{}: {e}", snake(name)))? + ]), + (Err(e), _) => Err(format!("{}: {e}", snake(name))), + }, + (_, Json::Object(map)) if has_all(map) => named(map), (_, Json::Array(items)) if items.len() == params.len() => params .iter() .zip(items) - .map(|((n, t), v)| to_val(t, v).map_err(|e| format!("{n}: {e}"))) + .map(|((n, t), v)| to_val(t, v).map_err(|e| format!("{}: {e}", snake(n)))) .collect(), _ => Err(format!( "takes {}: pass an object with those names", params .iter() - .map(|(n, _)| n.as_str()) + .map(|(n, _)| snake(n)) .collect::>() .join(", ") )), } } +/// The value under a WIT name, spelled either way. +fn field<'a>(map: &'a Map, wit_name: &str) -> Option<&'a Json> { + map.get(wit_name).or_else(|| map.get(&snake(wit_name))) +} + /// JSON to a WIT value of type `ty`. pub fn to_val(ty: &Type, v: &Json) -> Result { fn int + TryFrom>(v: &Json, what: &str) -> Result { @@ -686,7 +832,7 @@ pub fn to_val(ty: &Type, v: &Json) -> Result { record .fields() .map(|f| { - let value = map.get(f.name).unwrap_or(&Json::Null); + let value = field(map, f.name).unwrap_or(&Json::Null); to_val(&f.ty, value) .map(|val| (f.name.to_string(), val)) .map_err(|e| format!("{}: {e}", f.name)) @@ -725,23 +871,24 @@ pub fn to_val(ty: &Type, v: &Json) -> Result { }; let found = variant .cases() - .find(|c| c.name == case) + .find(|c| c.name == case || snake(c.name) == case) .ok_or_else(|| format!("no case {case}"))?; let payload = match (&found.ty, payload) { (Some(t), Some(p)) => Some(Box::new(to_val(t, p)?)), (Some(_), None) => return Err(format!("case {case} takes a value")), (None, _) => None, }; - Val::Variant(case.to_string(), payload) + Val::Variant(found.name.to_string(), payload) } Type::Enum(e) => { let name = v .as_str() .ok_or_else(|| format!("expected a name, got {v}"))?; - if !e.names().any(|n| n == name) { - return Err(format!("no case {name}")); - } - Val::Enum(name.to_string()) + let wit_name = e + .names() + .find(|n| *n == name || snake(n) == name) + .ok_or_else(|| format!("no case {name}"))?; + Val::Enum(wit_name.to_string()) } Type::Option(opt) => match v { Json::Null => Val::Option(None), @@ -773,10 +920,11 @@ pub fn to_val(ty: &Type, v: &Json) -> Result { let name = item .as_str() .ok_or_else(|| format!("expected a name, got {item}"))?; - if !f.names().any(|n| n == name) { - return Err(format!("no flag {name}")); - } - out.push(name.to_string()); + let wit_name = f + .names() + .find(|n| *n == name || snake(n) == name) + .ok_or_else(|| format!("no flag {name}"))?; + out.push(wit_name.to_string()); } Val::Flags(out) } @@ -816,7 +964,7 @@ pub fn from_val(ty: &Type, v: &Val) -> Json { (Type::Record(record), Val::Record(fields)) => { let mut map = Map::new(); for (f, (k, val)) in record.fields().zip(fields) { - map.insert(k.clone(), from_val(&f.ty, val)); + map.insert(snake(k), from_val(&f.ty, val)); } Json::Object(map) } @@ -830,11 +978,11 @@ pub fn from_val(ty: &Type, v: &Val) -> Json { (Type::Variant(variant), Val::Variant(case, payload)) => { let ty = variant.cases().find(|c| c.name == case).and_then(|c| c.ty); match (ty, payload) { - (Some(t), Some(p)) => json!({ case.clone(): from_val(&t, p) }), - _ => json!(case), + (Some(t), Some(p)) => json!({ snake(case): from_val(&t, p) }), + _ => json!(snake(case)), } } - (_, Val::Enum(name)) => json!(name), + (_, Val::Enum(name)) => json!(snake(name)), (Type::Option(opt), Val::Option(o)) => o .as_deref() .map(|v| from_val(&opt.ty(), v)) @@ -843,7 +991,7 @@ pub fn from_val(ty: &Type, v: &Val) -> Json { Ok(v) => json!({"ok": v.as_deref().zip(res.ok()).map(|(v, t)| from_val(&t, v))}), Err(e) => json!({"err": e.as_deref().zip(res.err()).map(|(v, t)| from_val(&t, v))}), }, - (_, Val::Flags(names)) => json!(names), + (_, Val::Flags(names)) => json!(names.iter().map(|n| snake(n)).collect::>()), _ => Json::Null, } } diff --git a/crates/wasm-host/src/component/files.rs b/crates/wasm-host/src/component/files.rs new file mode 100644 index 000000000..081f81389 --- /dev/null +++ b/crates/wasm-host/src/component/files.rs @@ -0,0 +1,327 @@ +//! A component's storage folder under a byte budget, and its file writes +//! done on the calling thread. +//! +//! wasmtime-wasi gives a component its folder ([`super::Grants`]). This +//! replaces the calls that grow a file — `write`, `set-size`, and the +//! streams `write-via-stream` and `append-via-stream` hand out — with ones +//! that charge the growth against the call's [`Budget`], and the calls that +//! free bytes — `open-at` truncating a file, `unlink-file-at` — with ones +//! that give them back. Growth past the budget fails inside the guest: as a +//! full disk (`ENOSPC`) from a descriptor call, and as an I/O error (`EIO`) +//! from a stream, since wasi-libc reports any failed stream write as one. +//! The budget records the refusal, so the caller can say why. Overwriting, +//! truncating and deleting always work, so a component whose app is at its +//! quota can still free space. +//! +//! What counts is what the shell's storage measure counts: the lengths of +//! the folder's regular files. + +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use wasmtime::component::{Linker, Resource}; +use wasmtime::StoreContextMut; +use wasmtime_wasi::filesystem::WasiFilesystemView; +use wasmtime_wasi::p2::bindings::sync::filesystem::types::{ + self as fs, Host as _, HostDescriptor as _, +}; +use wasmtime_wasi::p2::{DynOutputStream, OutputStream, Pollable, StreamError, StreamResult}; +use wasmtime_wasi::runtime::in_tokio; + +use super::State; + +/// The filesystem interface wasmtime-wasi links; a guest's older 0.2 +/// imports resolve to it. +const TYPES: &str = "wasi:filesystem/types@0.2.12"; + +/// A full disk, to the guest: `ENOSPC` where WASI maps it (Linux, Android, +/// macOS, iOS); elsewhere an error of that kind. +fn full() -> std::io::Error { + #[cfg(unix)] + { + std::io::Error::from_raw_os_error(28) + } + #[cfg(not(unix))] + { + std::io::Error::new( + std::io::ErrorKind::StorageFull, + "the storage budget is used up", + ) + } +} + +/// What an instance may still add to its storage folder (`None`: no +/// ceiling), shared with the file streams it has open. +#[derive(Clone, Debug, Default)] +pub(crate) struct Budget(Arc>); + +#[derive(Debug, Default)] +struct Ledger { + left: Option, + /// A write was refused since the caller last asked. + refused: bool, +} + +impl Budget { + fn ledger(&self) -> std::sync::MutexGuard<'_, Ledger> { + self.0.lock().unwrap_or_else(|e| e.into_inner()) + } + + pub(crate) fn set(&self, left: Option) { + self.ledger().left = left; + } + + pub(crate) fn left(&self) -> Option { + self.ledger().left + } + + /// Whether a write was refused since the last time this was asked. + pub(crate) fn take_refused(&self) -> bool { + std::mem::take(&mut self.ledger().refused) + } + + /// Spend `bytes`, or refuse them as a full disk. + fn spend(&self, bytes: u64) -> std::io::Result<()> { + let mut ledger = self.ledger(); + match ledger.left { + Some(n) if bytes > n => { + ledger.refused = true; + Err(full()) + } + Some(n) => { + ledger.left = Some(n - bytes); + Ok(()) + } + None => Ok(()), + } + } + + fn refund(&self, bytes: u64) { + if let Some(n) = self.ledger().left.as_mut() { + *n = n.saturating_add(bytes); + } + } +} + +type Answer = wasmtime::Result<(Result,)>; + +/// Shadow the linker's growing and freeing filesystem calls with charged +/// ones. Call after `wasmtime_wasi::p2::add_to_linker_sync`. +pub(crate) fn link(linker: &mut Linker) -> wasmtime::Result<()> { + linker.allow_shadowing(true); + let mut types = linker.instance(TYPES)?; + types.func_wrap( + "[method]descriptor.write-via-stream", + |mut store: StoreContextMut<'_, State>, (fd, offset): (Resource, u64)| { + charged_stream(store.data_mut(), fd, Some(offset)) + }, + )?; + types.func_wrap( + "[method]descriptor.append-via-stream", + |mut store: StoreContextMut<'_, State>, (fd,): (Resource,)| { + charged_stream(store.data_mut(), fd, None) + }, + )?; + types.func_wrap( + "[method]descriptor.write", + |mut store: StoreContextMut<'_, State>, + (fd, buffer, offset): (Resource, Vec, u64)| + -> Answer { + let state = store.data_mut(); + let budget = state.budget.clone(); + let mut files = state.filesystem(); + let size = match files.stat(borrow(&fd)) { + Ok(stat) => stat.size, + Err(e) => return Ok((Err(files.convert_error_code(e)?),)), + }; + let grows = offset + .saturating_add(buffer.len() as u64) + .saturating_sub(size); + if budget.spend(grows).is_err() { + return Ok((Err(fs::ErrorCode::InsufficientSpace),)); + } + match files.write(fd, buffer, offset) { + Ok(written) => Ok((Ok(written),)), + Err(e) => { + budget.refund(grows); + Ok((Err(files.convert_error_code(e)?),)) + } + } + }, + )?; + types.func_wrap( + "[method]descriptor.set-size", + |mut store: StoreContextMut<'_, State>, + (fd, size): (Resource, u64)| + -> Answer<()> { + let state = store.data_mut(); + let budget = state.budget.clone(); + let mut files = state.filesystem(); + let was = match files.stat(borrow(&fd)) { + Ok(stat) => stat.size, + Err(e) => return Ok((Err(files.convert_error_code(e)?),)), + }; + let grows = size.saturating_sub(was); + if budget.spend(grows).is_err() { + return Ok((Err(fs::ErrorCode::InsufficientSpace),)); + } + match files.set_size(fd, size) { + Ok(()) => { + budget.refund(was.saturating_sub(size)); + Ok((Ok(()),)) + } + Err(e) => { + budget.refund(grows); + Ok((Err(files.convert_error_code(e)?),)) + } + } + }, + )?; + types.func_wrap( + "[method]descriptor.open-at", + |mut store: StoreContextMut<'_, State>, + (fd, path_flags, path, open_flags, flags): ( + Resource, + fs::PathFlags, + String, + fs::OpenFlags, + fs::DescriptorFlags, + )| + -> Answer> { + let state = store.data_mut(); + let budget = state.budget.clone(); + let mut files = state.filesystem(); + let freed = if open_flags.contains(fs::OpenFlags::TRUNCATE) { + file_size_at(&mut files, &fd, path_flags, &path) + } else { + 0 + }; + match files.open_at(fd, path_flags, path, open_flags, flags) { + Ok(opened) => { + budget.refund(freed); + Ok((Ok(opened),)) + } + Err(e) => Ok((Err(files.convert_error_code(e)?),)), + } + }, + )?; + types.func_wrap( + "[method]descriptor.unlink-file-at", + |mut store: StoreContextMut<'_, State>, + (fd, path): (Resource, String)| + -> Answer<()> { + let state = store.data_mut(); + let budget = state.budget.clone(); + let mut files = state.filesystem(); + let freed = file_size_at(&mut files, &fd, fs::PathFlags::empty(), &path); + match files.unlink_file_at(fd, path) { + Ok(()) => { + budget.refund(freed); + Ok((Ok(()),)) + } + Err(e) => Ok((Err(files.convert_error_code(e)?),)), + } + }, + )?; + linker.allow_shadowing(false); + Ok(()) +} + +/// Another handle on the descriptor `fd` borrows, for a call before the +/// one it is passed to. +fn borrow(fd: &Resource) -> Resource { + Resource::new_borrow(fd.rep()) +} + +/// The bytes the regular file at `path` holds, as the storage measure +/// counts them (0 for anything else, or nothing there). +fn file_size_at( + files: &mut V, + fd: &Resource, + path_flags: fs::PathFlags, + path: &str, +) -> u64 { + match files.stat_at(borrow(fd), path_flags, path.to_string()) { + Ok(stat) if stat.type_ == fs::DescriptorType::RegularFile => stat.size, + _ => 0, + } +} + +/// The stream `write-via-stream` (`offset`) or `append-via-stream` +/// (`None`) hands out, charged. +fn charged_stream( + state: &mut State, + fd: Resource, + offset: Option, +) -> Answer> { + let budget = state.budget.clone(); + let mut files = state.filesystem(); + let size = match files.stat(borrow(&fd)) { + Ok(stat) => stat.size, + Err(e) => return Ok((Err(files.convert_error_code(e)?),)), + }; + let opened = match offset { + Some(offset) => files.write_via_stream(fd, offset), + None => files.append_via_stream(fd), + }; + let stream = match opened { + Ok(stream) => stream, + Err(e) => return Ok((Err(files.convert_error_code(e)?),)), + }; + let inner = files.table.delete(stream)?; + let charged: DynOutputStream = Box::new(Charged { + inner, + budget, + position: offset, + size, + }); + Ok((Ok(files.table.push(charged)?),)) +} + +/// A file's output stream that charges what it adds to the file, and +/// writes on the calling thread: the plain stream hands every write to a +/// thread pool and waits for it on the next call. +struct Charged { + inner: DynOutputStream, + budget: Budget, + /// Where the next write lands; `None` when appending. + position: Option, + /// The file's size as far as this stream knows. + size: u64, +} + +#[wasmtime_wasi::async_trait] +impl Pollable for Charged { + async fn ready(&mut self) { + self.inner.ready().await + } +} + +#[wasmtime_wasi::async_trait] +impl OutputStream for Charged { + fn write(&mut self, bytes: Bytes) -> StreamResult<()> { + let len = bytes.len() as u64; + let end = self.position.unwrap_or(self.size).saturating_add(len); + let grows = end.saturating_sub(self.size); + self.budget + .spend(grows) + .map_err(|e| StreamError::LastOperationFailed(e.into()))?; + if let Err(e) = in_tokio(self.inner.blocking_write_and_flush(bytes)) { + self.budget.refund(grows); + return Err(e); + } + if let Some(position) = &mut self.position { + *position = end; + } + self.size = self.size.max(end); + Ok(()) + } + + fn flush(&mut self) -> StreamResult<()> { + self.inner.flush() + } + + fn check_write(&mut self) -> StreamResult { + self.inner.check_write() + } +} diff --git a/crates/wasm-host/src/component/host.rs b/crates/wasm-host/src/component/host.rs new file mode 100644 index 000000000..4d936e606 --- /dev/null +++ b/crates/wasm-host/src/component/host.rs @@ -0,0 +1,53 @@ +//! `octosense:host`: a component calls its app's host services, as the +//! app's script does with `host.request` (`wit/octosense-host.wit`). +//! +//! The runtime knows no services: the embedder hands an instance its app's +//! [`HostCalls`], which decides what the app may call and makes the call. +//! Without one, every call answers that this host has none. A call waits +//! outside the guest, where the deadline's epoch check cannot end it, so +//! [`HostCalls::request`] gets the call's deadline and must answer by then. + +use std::sync::Arc; +use std::time::Instant; + +use wasmtime::component::Linker; +use wasmtime::StoreContextMut; + +use super::State; + +/// The interface a component imports (older 0.1 patch versions resolve to +/// it). +pub(crate) const INTERFACE: &str = "octosense:host/services@0.1.0"; + +/// An app's host services, as a component reaches them. +pub trait HostCalls: Send + Sync { + /// Calls `service` (`family.method`) with JSON `args`: its JSON answer, + /// or why there is none. Answers by `deadline`. + fn request(&self, service: &str, args: &str, deadline: Instant) -> Result; +} + +/// Links `octosense:host/services`. +pub(crate) fn link(linker: &mut Linker) -> wasmtime::Result<()> { + linker.instance(INTERFACE)?.func_wrap( + "request", + |store: StoreContextMut<'_, State>, + (service, args): (String, String)| + -> wasmtime::Result<(Result,)> { + let state = store.data(); + let deadline = state + .guard + .as_ref() + .map(|guard| guard.deadline) + .unwrap_or_else(Instant::now); + let answer = match &state.host_calls { + Some(host) => host.request(&service, &args, deadline), + None => Err("this host gives a component no host services".into()), + }; + Ok((answer,)) + }, + )?; + Ok(()) +} + +/// The calls an instance's host services get, set by the embedder. +pub(crate) type Calls = Option>; diff --git a/crates/wasm-host/src/component/net.rs b/crates/wasm-host/src/component/net.rs new file mode 100644 index 000000000..fb8790faf --- /dev/null +++ b/crates/wasm-host/src/component/net.rs @@ -0,0 +1,92 @@ +//! A component's outgoing `wasi:http`: to any host. +//! +//! An app's network declarations (`net`, `network.hosts`) are shown when it +//! is installed, not enforced while it runs: the architecture ruling of +//! 8 October 2026 removes the per-app runtime gates, and the OS and the +//! host's API surface are the boundary. (The app's Splash script still +//! passes the Splash runtime's URL gate until the Makepad fork drops it; a +//! component never gets one.) +//! +//! What the runtime still owes the call: a request waits for the network +//! outside the guest, where the deadline's epoch check cannot end it, so +//! its timeouts (connect, first byte, between bytes) are clamped to what is +//! left of the call. + +use std::future::Future; +use std::time::{Duration, Instant}; + +use http_body_util::BodyExt; +use wasmtime_wasi_http::{Error, RequestOptions, WasiBody, WasiHttpHooks}; + +/// The running call's deadline, for the requests it sends. +#[derive(Debug, Default)] +pub(crate) struct Network { + /// The running call's deadline, set for every call. + pub(crate) deadline: Option, +} + +type Response = ( + http::Response, + Box> + Send>, +); + +impl WasiHttpHooks for Network { + fn send_request( + &mut self, + request: http::Request, + options: Option, + fut: Box> + Send>, + ) -> Box> + Send> { + _ = fut; + let left = self + .deadline + .map(|deadline| deadline.saturating_duration_since(Instant::now())); + Box::new(async move { + let options = clamped(options.unwrap_or_default(), left); + let (response, io) = + wasmtime_wasi_http::default_send_request(request, Some(options)).await?; + Ok(( + response.map(BodyExt::boxed_unsync), + Box::new(io) as Box> + Send>, + )) + }) + } +} + +/// `options` with no timeout longer than `left` (and `left` where it sets +/// none). +fn clamped(options: RequestOptions, left: Option) -> RequestOptions { + let clamp = |asked: Option| match (asked, left) { + (Some(asked), Some(left)) => Some(asked.min(left)), + (asked, left) => asked.or(left), + }; + RequestOptions { + connect_timeout: clamp(options.connect_timeout), + first_byte_timeout: clamp(options.first_byte_timeout), + between_bytes_timeout: clamp(options.between_bytes_timeout), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// No timeout outlives the call. + #[test] + fn a_requests_timeouts_end_with_the_call() { + let left = Some(Duration::from_secs(3)); + let asked = RequestOptions { + connect_timeout: Some(Duration::from_secs(60)), + first_byte_timeout: Some(Duration::from_secs(1)), + between_bytes_timeout: None, + }; + let got = clamped(asked, left); + assert_eq!(got.connect_timeout, Some(Duration::from_secs(3))); + assert_eq!(got.first_byte_timeout, Some(Duration::from_secs(1))); + assert_eq!(got.between_bytes_timeout, Some(Duration::from_secs(3))); + assert_eq!( + clamped(RequestOptions::default(), None).connect_timeout, + None + ); + } +} diff --git a/crates/wasm-host/src/lib.rs b/crates/wasm-host/src/lib.rs index ee83f29f3..68f6b0312 100644 --- a/crates/wasm-host/src/lib.rs +++ b/crates/wasm-host/src/lib.rs @@ -56,6 +56,9 @@ const LOG_LINES: usize = 64; pub struct Limits { /// How long one call may run. pub deadline: Duration, + /// How long one call may run when it may wait for the network (a + /// component that imports `wasi:http`). + pub network_deadline: Duration, /// The most linear memory an instance may have. pub memory_bytes: usize, /// The most elements in an instance's single table (including growth). @@ -72,6 +75,7 @@ impl Default for Limits { fn default() -> Self { Limits { deadline: Duration::from_secs(2), + network_deadline: Duration::from_secs(10), memory_bytes: 256 << 20, table_elements: 16_384, module_bytes: 8 << 20, @@ -215,6 +219,11 @@ impl Runtime { // Components (ADR 0014) share the engine, its epoch and its cache. config.wasm_component_model(true); config.max_wasm_stack(limits.stack_bytes); + if Runtime::interpreted() { + config + .target("pulley64") + .map_err(|e| format!("Pulley: {e:#}"))?; + } let engine = Engine::new(&config).map_err(|e| format!("{e:#}"))?; let ticker = Arc::new(Ticker::start(engine.clone())); Ok(Runtime { @@ -229,6 +238,18 @@ impl Runtime { &self.limits } + /// Whether code runs in Pulley, Wasmtime's interpreter, rather than as + /// native code Cranelift generates (ADR 0014 phase 3): on iOS, where an + /// app may not generate code, and on OpenHarmony, until its policy is + /// known. With the `pulley` feature, `OCTOSENSE_WASM_PULLEY` selects it + /// on any host, so that its tests run anywhere. Cranelift then compiles + /// to Pulley's bytecode, and the cache keeps it apart (the engine's + /// compatibility hash differs). + pub fn interpreted() -> bool { + cfg!(any(target_os = "ios", target_env = "ohos")) + || (cfg!(feature = "pulley") && std::env::var_os("OCTOSENSE_WASM_PULLEY").is_some()) + } + /// Checks and compiles `bytes`, or takes its compiled code from the cache. pub fn load(&self, bytes: &[u8]) -> Result { if bytes.len() > self.limits.module_bytes { @@ -257,6 +278,22 @@ impl Runtime { }) } + /// Compiles `bytes`, a module or a component, into the cache unless it + /// is there already; whether it compiled. An app's install calls this so + /// that its first call loads from the cache instead of waiting for + /// Cranelift (ADR 0014 phase 3). Without a cache it only checks. + pub fn precompile(&self, bytes: &[u8]) -> Result { + if self.cache_path(bytes).is_some_and(|path| path.is_file()) { + return Ok(false); + } + if component::is_component(bytes) { + self.load_component(bytes) + .map(|program| !program.from_cache()) + } else { + self.load(bytes).map(|program| !program.from_cache()) + } + } + fn compile(&self, bytes: &[u8], cache: Option<&PathBuf>) -> Result { let module = Module::new(&self.engine, bytes).map_err(|e| LoadError::Invalid(format!("{e:#}")))?; diff --git a/crates/wasm-host/tests/component-guest/Cargo.lock b/crates/wasm-host/tests/component-guest/Cargo.lock index f98eb5a23..69024f09f 100644 --- a/crates/wasm-host/tests/component-guest/Cargo.lock +++ b/crates/wasm-host/tests/component-guest/Cargo.lock @@ -26,6 +26,14 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fetch" +version = "0.1.0" +dependencies = [ + "wasi", + "wit-bindgen 0.62.0", +] + [[package]] name = "foldhash" version = "0.2.0" @@ -58,6 +66,13 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hostcall" +version = "0.1.0" +dependencies = [ + "wit-bindgen 0.62.0", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -121,7 +136,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" name = "netprobe" version = "0.1.0" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.62.0", ] [[package]] @@ -130,7 +145,7 @@ version = "0.1.0" dependencies = [ "getrandom", "pulldown-cmark", - "wit-bindgen", + "wit-bindgen 0.62.0", ] [[package]] @@ -256,6 +271,24 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + [[package]] name = "wasm-encoder" version = "0.259.0" @@ -290,6 +323,15 @@ dependencies = [ "semver", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +dependencies = [ + "bitflags", +] + [[package]] name = "wit-bindgen" version = "0.62.0" diff --git a/crates/wasm-host/tests/component-guest/Cargo.toml b/crates/wasm-host/tests/component-guest/Cargo.toml index 3c5598b37..57fd43c62 100644 --- a/crates/wasm-host/tests/component-guest/Cargo.toml +++ b/crates/wasm-host/tests/component-guest/Cargo.toml @@ -3,7 +3,7 @@ # ../fixtures. Its own workspace: none of this enters the shell's graph. [workspace] resolver = "2" -members = ["notes", "netprobe"] +members = ["notes", "netprobe", "fetch", "hostcall"] [profile.release] opt-level = "s" diff --git a/crates/wasm-host/tests/component-guest/build.sh b/crates/wasm-host/tests/component-guest/build.sh index b689bc75b..e4144ae9d 100755 --- a/crates/wasm-host/tests/component-guest/build.sh +++ b/crates/wasm-host/tests/component-guest/build.sh @@ -5,7 +5,7 @@ set -euo pipefail cd "$(dirname "$0")" cargo build --locked --target wasm32-wasip2 --release mkdir -p ../fixtures -for c in notes netprobe; do +for c in notes netprobe fetch hostcall; do cp "target/wasm32-wasip2/release/$c.wasm" "../fixtures/$c.component.wasm" done ls -l ../fixtures/*.component.wasm diff --git a/crates/wasm-host/tests/component-guest/fetch/Cargo.toml b/crates/wasm-host/tests/component-guest/fetch/Cargo.toml new file mode 100644 index 000000000..e299f5f45 --- /dev/null +++ b/crates/wasm-host/tests/component-guest/fetch/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "fetch" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.62" +# WASI 0.2's own bindings: the HTTP client a component uses. +wasi = "0.14.7" diff --git a/crates/wasm-host/tests/component-guest/fetch/src/lib.rs b/crates/wasm-host/tests/component-guest/fetch/src/lib.rs new file mode 100644 index 000000000..d8aa08896 --- /dev/null +++ b/crates/wasm-host/tests/component-guest/fetch/src/lib.rs @@ -0,0 +1,46 @@ +//! A test component that reaches the network over `wasi:http`, the way a +//! crate built for wasm32-wasip2 does, with WASI 0.2's own bindings. +use wasi::http::outgoing_handler; +use wasi::http::types::{Fields, Method, OutgoingRequest, Scheme}; +use wasi::io::streams::StreamError; + +wit_bindgen::generate!({ world: "fetch", path: "wit" }); + +struct Fetch; + +impl Guest for Fetch { + fn get(url: String) -> Result { + let (scheme, rest) = url.split_once("://").ok_or("the URL has no scheme")?; + let (authority, path) = match rest.find('/') { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, "/"), + }; + let request = OutgoingRequest::new(Fields::new()); + request.set_method(&Method::Get).map_err(|()| "method")?; + let scheme = if scheme == "https" { Scheme::Https } else { Scheme::Http }; + request.set_scheme(Some(&scheme)).map_err(|()| "scheme")?; + request.set_authority(Some(authority)).map_err(|()| "authority")?; + request.set_path_with_query(Some(path)).map_err(|()| "path")?; + let pending = outgoing_handler::handle(request, None).map_err(|e| format!("{e:?}"))?; + pending.subscribe().block(); + let response = pending + .get() + .ok_or("no response")? + .map_err(|()| "the response was taken")? + .map_err(|e| format!("{e:?}"))?; + let status = response.status(); + let body = response.consume().map_err(|()| "no body")?; + let stream = body.stream().map_err(|()| "no body stream")?; + let mut bytes = Vec::new(); + loop { + match stream.blocking_read(64 << 10) { + Ok(chunk) => bytes.extend_from_slice(&chunk), + Err(StreamError::Closed) => break, + Err(e) => return Err(format!("{e:?}")), + } + } + Ok(format!("{status} {}", String::from_utf8_lossy(&bytes))) + } +} + +export!(Fetch); diff --git a/crates/wasm-host/tests/component-guest/fetch/wit/fetch.wit b/crates/wasm-host/tests/component-guest/fetch/wit/fetch.wit new file mode 100644 index 000000000..b080fca0d --- /dev/null +++ b/crates/wasm-host/tests/component-guest/fetch/wit/fetch.wit @@ -0,0 +1,6 @@ +package octosense:fetch-spike; + +world fetch { + /// GET `url` over wasi:http: " ", or the error. + export get: func(url: string) -> result; +} diff --git a/crates/wasm-host/tests/component-guest/hostcall/Cargo.toml b/crates/wasm-host/tests/component-guest/hostcall/Cargo.toml new file mode 100644 index 000000000..801462285 --- /dev/null +++ b/crates/wasm-host/tests/component-guest/hostcall/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "hostcall" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.62" diff --git a/crates/wasm-host/tests/component-guest/hostcall/src/lib.rs b/crates/wasm-host/tests/component-guest/hostcall/src/lib.rs new file mode 100644 index 000000000..9f4329c27 --- /dev/null +++ b/crates/wasm-host/tests/component-guest/hostcall/src/lib.rs @@ -0,0 +1,13 @@ +//! A test component that calls its app's host services through +//! `octosense:host` (crates/wasm-host/wit/octosense-host.wit). +wit_bindgen::generate!({ world: "hostcall", path: "wit", generate_all }); + +struct HostCall; + +impl Guest for HostCall { + fn call(service: String, args: String) -> Result { + octosense::host::services::request(&service, &args) + } +} + +export!(HostCall); diff --git a/crates/wasm-host/tests/component-guest/hostcall/wit/deps/octosense-host/host.wit b/crates/wasm-host/tests/component-guest/hostcall/wit/deps/octosense-host/host.wit new file mode 100644 index 000000000..e3bbf27cb --- /dev/null +++ b/crates/wasm-host/tests/component-guest/hostcall/wit/deps/octosense-host/host.wit @@ -0,0 +1,14 @@ +package octosense:host@0.1.0; + +/// The app's host services, as its script's `host.request` reaches them: +/// only the families its app is granted, and never a sheet or a prompt. +interface services { + /// Calls `service` ("family.method") with JSON `args`: the service's JSON + /// answer, or why there is none. + request: func(service: string, args: string) -> result; +} + +/// What a component imports to reach them. +world component-host { + import services; +} diff --git a/crates/wasm-host/tests/component-guest/hostcall/wit/hostcall.wit b/crates/wasm-host/tests/component-guest/hostcall/wit/hostcall.wit new file mode 100644 index 000000000..f53a89b36 --- /dev/null +++ b/crates/wasm-host/tests/component-guest/hostcall/wit/hostcall.wit @@ -0,0 +1,7 @@ +package octosense:hostcall-spike; + +world hostcall { + import octosense:host/services@0.1.0; + /// Forwards to the app's host services. + export call: func(service: string, args: string) -> result; +} diff --git a/crates/wasm-host/tests/component-guest/notes/src/lib.rs b/crates/wasm-host/tests/component-guest/notes/src/lib.rs index dde29f7a1..625de1745 100644 --- a/crates/wasm-host/tests/component-guest/notes/src/lib.rs +++ b/crates/wasm-host/tests/component-guest/notes/src/lib.rs @@ -84,6 +84,43 @@ impl Guest for Notes { fn echo_bytes(data: Vec) -> Vec { data } + + fn append_text(path: String, text: String) -> Result { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .append(true) + .create(true) + .open(&path) + .map_err(|e| e.to_string())?; + file.write_all(text.as_bytes()).map_err(|e| e.to_string())?; + Ok(file.metadata().map_err(|e| e.to_string())?.len()) + } + + fn set_len(path: String, len: u64) -> Result<(), String> { + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .and_then(|file| file.set_len(len)) + .map_err(|e| e.to_string()) + } + + fn delete_file(path: String) -> Result<(), String> { + std::fs::remove_file(&path).map_err(|e| e.to_string()) + } + + fn file_info(path: String) -> Result { + let meta = std::fs::metadata(&path).map_err(|e| e.to_string())?; + Ok(FileInfo { + byte_count: meta.len(), + kind: if meta.is_file() { + EntryKind::RegularFile + } else if meta.is_dir() { + EntryKind::Directory + } else { + EntryKind::Other + }, + }) + } } export!(Notes); diff --git a/crates/wasm-host/tests/component-guest/notes/wit/notes.wit b/crates/wasm-host/tests/component-guest/notes/wit/notes.wit index ba41e55df..573fac2f3 100644 --- a/crates/wasm-host/tests/component-guest/notes/wit/notes.wit +++ b/crates/wasm-host/tests/component-guest/notes/wit/notes.wit @@ -25,4 +25,19 @@ world notes { export grow: func(megabytes: u32) -> u32; /// Bytes in, bytes out. export echo-bytes: func(data: list) -> list; + + /// Appends to a file (append-via-stream); its new length. + export append-text: func(path: string, text: string) -> result; + /// Sets a file's length (set-size). + export set-len: func(path: string, len: u64) -> result<_, string>; + /// Deletes a file (unlink-file-at). + export delete-file: func(path: string) -> result<_, string>; + + enum entry-kind { regular-file, directory, other } + record file-info { + byte-count: u64, + kind: entry-kind, + } + /// Hyphenated names, which scripts read in snake_case. + export file-info: func(path: string) -> result; } diff --git a/crates/wasm-host/tests/component.rs b/crates/wasm-host/tests/component.rs index f17dbeb35..5de409b86 100644 --- a/crates/wasm-host/tests/component.rs +++ b/crates/wasm-host/tests/component.rs @@ -5,12 +5,15 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; +use std::sync::Arc; +use std::time::{Duration, Instant}; -use octosense_wasm_host::component::{is_component, Grants}; +use octosense_wasm_host::component::{is_component, Grants, HostCalls}; use octosense_wasm_host::{CallError, Limits, LoadError, Runtime}; use serde_json::json; +const HOSTCALL: &[u8] = include_bytes!("fixtures/hostcall.component.wasm"); +const FETCH: &[u8] = include_bytes!("fixtures/fetch.component.wasm"); const NOTES: &[u8] = include_bytes!("fixtures/notes.component.wasm"); const NETPROBE: &[u8] = include_bytes!("fixtures/netprobe.component.wasm"); @@ -53,15 +56,19 @@ fn exports_are_listed_with_their_wit_signatures() { names, [ "analyze", + "append_text", "count", - "echo-bytes", + "delete_file", + "echo_bytes", + "file_info", "grow", - "now-ms", - "random-u64", - "read-file", - "save-html", + "now_ms", + "random_u64", + "read_file", + "save_html", + "set_len", "spin", - "to-html" + "to_html" ] ); let analyze = &program.exports()[0]; @@ -83,7 +90,7 @@ fn an_unmodified_crate_runs_and_records_come_back_as_objects() { .instantiate_component(&program, &Grants::default(), None) .unwrap(); let html = instance - .call_json("to-html", &json!({"markdown": "# Hi\n\nThere *you*"})) + .call_json("to_html", &json!({"markdown": "# Hi\n\nThere *you*"})) .unwrap(); assert_eq!(html, json!("

Hi

\n

There you

\n")); // A one-parameter function also takes the bare value. @@ -94,6 +101,9 @@ fn an_unmodified_crate_runs_and_records_come_back_as_objects() { stats, json!({"words": 7, "lines": 3, "headings": ["Title", "Part"]}) ); + // And an array in parameter order, as every other function does. + let stats = instance.call_json("analyze", &json!(["# One"])).unwrap(); + assert_eq!(stats, json!({"words": 2, "lines": 1, "headings": ["One"]})); } #[test] @@ -118,19 +128,24 @@ fn clocks_randomness_and_bytes_work() { let mut instance = rt .instantiate_component(&program, &Grants::default(), None) .unwrap(); - let now = instance.call_json("now-ms", &json!(null)).unwrap(); + let now = instance.call_json("now_ms", &json!(null)).unwrap(); assert!(now.as_u64().unwrap() > 1_700_000_000_000, "{now}"); - let a = instance.call_json("random-u64", &json!(null)).unwrap(); - let b = instance.call_json("random-u64", &json!(null)).unwrap(); + let a = instance.call_json("random_u64", &json!(null)).unwrap(); + let b = instance.call_json("random_u64", &json!(null)).unwrap(); assert_ne!(a, b); // list travels as base64. let echoed = instance - .call_json("echo-bytes", &json!("AAEC/w==")) + .call_json("echo_bytes", &json!("AAEC/w==")) .unwrap(); assert_eq!(echoed, json!("AAEC/w==")); + // An array is the list itself, unless only as its one argument. + for args in [json!([0, 1, 2, 255]), json!([[0, 1, 2, 255]])] { + let echoed = instance.call_json("echo_bytes", &args).unwrap(); + assert_eq!(echoed, json!("AAEC/w=="), "{args}"); + } assert_eq!( instance - .call_json("echo-bytes", &json!([0, 1, 2, 255])) + .call_json("echo_bytes", &json!([0, 1, 2, 255])) .unwrap(), json!("AAEC/w==") ); @@ -143,11 +158,12 @@ fn files_live_only_in_the_granted_storage_folder() { let dir = storage(); let grants = Grants { storage_dir: Some(dir.clone()), + read_only: false, }; let mut instance = rt.instantiate_component(&program, &grants, None).unwrap(); let written = instance .call_json( - "save-html", + "save_html", &json!({"markdown": "# Saved", "path": "out.html"}), ) .unwrap(); @@ -158,28 +174,270 @@ fn files_live_only_in_the_granted_storage_folder() { ); assert_eq!( instance - .call_json("read-file", &json!("/out.html")) + .call_json("read_file", &json!("/out.html")) .unwrap(), json!("

Saved

\n") ); // Nothing outside the folder: escaping it fails inside the guest. for path in ["../../../../etc/hosts", "/../etc/hosts"] { - match instance.call_json("read-file", &json!(path)) { + match instance.call_json("read_file", &json!(path)) { Err(CallError::Guest(_)) => {} other => panic!("{path}: {other:?}"), } } + // Read-only (a used-up quota): its files read, nothing writes. + let read_only = Grants { + storage_dir: Some(dir.clone()), + read_only: true, + }; + let mut reader = rt + .instantiate_component(&program, &read_only, None) + .unwrap(); + assert_eq!( + reader.call_json("read_file", &json!("out.html")).unwrap(), + json!("

Saved

\n") + ); + for path in ["out.html", "new.html"] { + assert!(matches!( + reader.call_json("save_html", &json!({"markdown": "x", "path": path})), + Err(CallError::Guest(_)) + )); + } + assert!(!dir.join("new.html").exists()); + assert_eq!( + std::fs::read_to_string(dir.join("out.html")).unwrap(), + "

Saved

\n" + ); // Without a storage grant there is no filesystem at all. let mut bare = rt .instantiate_component(&program, &Grants::default(), None) .unwrap(); assert!(matches!( - bare.call_json("save-html", &json!({"markdown": "x", "path": "x.html"})), + bare.call_json("save_html", &json!({"markdown": "x", "path": "x.html"})), Err(CallError::Guest(_)) )); let _ = std::fs::remove_dir_all(dir); } +/// The storage budget: growth past it fails in the guest as a full disk, +/// while rewriting (truncating first), shrinking and deleting give bytes +/// back. A guest's error does not spend the instance. +#[test] +fn the_storage_budget_refuses_growth_and_returns_freed_bytes() { + let rt = runtime(); + let program = rt.load_component(NOTES).unwrap(); + let dir = storage(); + let grants = Grants { + storage_dir: Some(dir.clone()), + read_only: false, + }; + let mut instance = rt.instantiate_component(&program, &grants, None).unwrap(); + assert_eq!(instance.storage_budget(), None); + let full = |result: Result| match result { + Err(CallError::Guest(why)) => { + assert!(why.ends_with("the storage budget is used up"), "{why}") + } + other => panic!("expected a full disk, got {other:?}"), + }; + // "# Saved" renders to 15 bytes. + let save = |path: &str| json!({"markdown": "# Saved", "path": path}); + instance.set_storage_budget(Some(20)); + assert_eq!( + instance.call_json("save_html", &save("a.html")).unwrap(), + json!(15) + ); + assert_eq!(instance.storage_budget(), Some(5)); + // Another 15 bytes do not fit (write-via-stream); the file stays empty. + full(instance.call_json("save_html", &save("b.html"))); + assert_eq!(std::fs::metadata(dir.join("b.html")).unwrap().len(), 0); + assert!(!instance.spent()); + // Rewriting a.html truncates it first: its 15 bytes come back. + assert_eq!( + instance.call_json("save_html", &save("a.html")).unwrap(), + json!(15) + ); + assert_eq!(instance.storage_budget(), Some(5)); + // Appending (append-via-stream) charges every byte. + assert_eq!( + instance + .call_json("append_text", &json!(["a.html", "12345"])) + .unwrap(), + json!(20) + ); + full(instance.call_json("append_text", &json!(["a.html", "x"]))); + // Growing a file's length (set-size) is growth; shrinking gives back. + full(instance.call_json("set_len", &json!(["a.html", 1 << 30]))); + assert_eq!(std::fs::metadata(dir.join("a.html")).unwrap().len(), 20); + instance + .call_json("set_len", &json!(["a.html", 10])) + .unwrap(); + assert_eq!(instance.storage_budget(), Some(10)); + // Deleting (unlink-file-at) gives a file's bytes back. + instance.call_json("delete_file", &json!("a.html")).unwrap(); + assert_eq!(instance.storage_budget(), Some(20)); + assert_eq!( + instance.call_json("file_info", &json!("b.html")).unwrap(), + json!({"byte_count": 0, "kind": "regular_file"}) + ); + // Without a ceiling, anything. + instance.set_storage_budget(None); + assert_eq!( + instance + .call_json("append_text", &json!(["big.txt", "x".repeat(100_000)])) + .unwrap(), + json!(100_000) + ); + let _ = std::fs::remove_dir_all(dir); +} + +/// A local HTTP server: every connection gets `reply` (or, with `None`, an +/// accepted connection that never answers). Its `host:port`. +fn serve(reply: Option<&'static str>) -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming().flatten() { + let Some(body) = reply else { + held.push(stream); + continue; + }; + let mut stream = stream; + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if std::io::Read::read(&mut stream, &mut byte).unwrap_or(0) == 0 { + break; + } + request.push(byte[0]); + } + let answer = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = std::io::Write::write_all(&mut stream, answer.as_bytes()); + } + }); + address +} + +/// `wasi:http` reaches any host, as the app's script does: an app's network +/// declarations are shown at install, not enforced (the ruling of +/// 8 October 2026), and no grant is needed. +#[test] +fn http_reaches_any_host() { + let rt = runtime(); + let program = rt.load_component(FETCH).unwrap(); + let server = serve(Some("hello")); + let port = server.rsplit(':').next().unwrap().to_string(); + let mut instance = rt + .instantiate_component(&program, &Grants::default(), None) + .unwrap(); + assert_eq!( + instance + .call_json("get", &json!(format!("http://{server}/hi"))) + .unwrap(), + json!("200 hello") + ); + // The same server under another name: still reached. + assert_eq!( + instance + .call_json("get", &json!(format!("http://localhost:{port}/hi"))) + .unwrap(), + json!("200 hello") + ); + assert!( + instance.take_logs().is_empty(), + "nothing is refused or logged" + ); +} + +/// A request waits outside the guest, where the epoch check cannot end it: +/// its timeouts are clamped to the call's deadline, so a server that never +/// answers ends the call instead of holding the worker. +#[test] +fn a_request_that_never_answers_ends_at_the_deadline() { + let rt = Runtime::new( + Limits { + network_deadline: Duration::from_millis(800), + ..Limits::default() + }, + None, + ) + .unwrap(); + let program = rt.load_component(FETCH).unwrap(); + let server = serve(None); + let mut instance = rt + .instantiate_component(&program, &Grants::default(), None) + .unwrap(); + let started = std::time::Instant::now(); + let result = instance.call_json("get", &json!(format!("http://{server}/"))); + assert!(result.is_err(), "{result:?}"); + assert!( + started.elapsed() < Duration::from_secs(5), + "ended after {:?}", + started.elapsed() + ); +} + +/// An embedder's host services for the `hostcall` component: it records +/// each call, answers `notes.get`, refuses the rest, and makes `slow.wait` +/// wait out the call's deadline. +#[derive(Default)] +struct Services(std::sync::Mutex>); + +impl HostCalls for Services { + fn request(&self, service: &str, args: &str, deadline: Instant) -> Result { + self.0.lock().unwrap().push((service.into(), args.into())); + match service { + "notes.get" => Ok(format!(r#"{{"echo":{args}}}"#)), + "slow.wait" => { + std::thread::sleep(deadline.saturating_duration_since(Instant::now())); + Err("past the call's deadline".into()) + } + _ => Err(format!("this app was not granted {service}")), + } + } +} + +/// `octosense:host`: a component calls its app's host services through the +/// embedder, which decides what it may call; without one it reaches none, +/// and a call's deadline reaches the embedder. +#[test] +fn a_component_calls_its_apps_host_services_through_the_embedder() { + let rt = runtime(); + let program = rt.load_component(HOSTCALL).unwrap(); + let mut instance = rt + .instantiate_component(&program, &Grants::default(), None) + .unwrap(); + match instance.call_json("call", &json!(["notes.get", "{}"])) { + Err(CallError::Guest(why)) => assert!(why.contains("no host services"), "{why}"), + other => panic!("{other:?}"), + } + let services = Arc::new(Services::default()); + instance.set_host_calls(Some(services.clone())); + assert_eq!( + instance + .call_json("call", &json!(["notes.get", r#"{"id":1}"#])) + .unwrap(), + json!(r#"{"echo":{"id":1}}"#) + ); + match instance.call_json("call", &json!(["mail.send", "{}"])) { + Err(CallError::Guest(why)) => assert!(why.contains("not granted mail.send"), "{why}"), + other => panic!("{other:?}"), + } + let started = Instant::now(); + assert!(instance + .call_json("call", &json!(["slow.wait", "{}"])) + .is_err()); + assert!( + started.elapsed() < Duration::from_secs(3), + "{:?}", + started.elapsed() + ); + assert_eq!(services.0.lock().unwrap().len(), 3); +} + #[test] fn the_deadline_and_the_memory_cap_end_a_call_and_spend_the_instance() { let rt = runtime(); @@ -212,7 +470,7 @@ fn arguments_that_do_not_fit_are_refused_with_the_parameter_name() { let mut instance = rt .instantiate_component(&program, &Grants::default(), None) .unwrap(); - match instance.call_json("save-html", &json!({"markdown": 3, "path": "x"})) { + match instance.call_json("save_html", &json!({"markdown": 3, "path": "x"})) { Err(CallError::Guest(why)) => assert!(why.starts_with("markdown:"), "{why}"), other => panic!("{other:?}"), } diff --git a/crates/wasm-host/tests/fixtures/fetch.component.wasm b/crates/wasm-host/tests/fixtures/fetch.component.wasm new file mode 100644 index 000000000..3356c178d Binary files /dev/null and b/crates/wasm-host/tests/fixtures/fetch.component.wasm differ diff --git a/crates/wasm-host/tests/fixtures/hostcall.component.wasm b/crates/wasm-host/tests/fixtures/hostcall.component.wasm new file mode 100644 index 000000000..c0ce9239b Binary files /dev/null and b/crates/wasm-host/tests/fixtures/hostcall.component.wasm differ diff --git a/crates/wasm-host/tests/fixtures/notes.component.wasm b/crates/wasm-host/tests/fixtures/notes.component.wasm index 04b7472a2..ced7a5b6a 100644 Binary files a/crates/wasm-host/tests/fixtures/notes.component.wasm and b/crates/wasm-host/tests/fixtures/notes.component.wasm differ diff --git a/crates/wasm-host/wit/octosense-host.wit b/crates/wasm-host/wit/octosense-host.wit new file mode 100644 index 000000000..e3bbf27cb --- /dev/null +++ b/crates/wasm-host/wit/octosense-host.wit @@ -0,0 +1,14 @@ +package octosense:host@0.1.0; + +/// The app's host services, as its script's `host.request` reaches them: +/// only the families its app is granted, and never a sheet or a prompt. +interface services { + /// Calls `service` ("family.method") with JSON `args`: the service's JSON + /// answer, or why there is none. + request: func(service: string, args: string) -> result; +} + +/// What a component imports to reach them. +world component-host { + import services; +} diff --git a/docs/adr/0014-app-components-in-webassembly.md b/docs/adr/0014-app-components-in-webassembly.md index 3d5e69ce6..429968c99 100644 --- a/docs/adr/0014-app-components-in-webassembly.md +++ b/docs/adr/0014-app-components-in-webassembly.md @@ -3,8 +3,14 @@ English | [简体中文](0014-app-components-in-webassembly.zh-CN.md) Status: Proposed (9 Oct 2026). Phase 1, the runtime spike, is in -`crates/wasm-host` (`src/component.rs`, `tests/component.rs`). Phases 2–4 below -are the plan. It extends [ADR 0011](0011-apps-own-functions-in-webassembly.md): +`crates/wasm-host` (`src/component.rs`, `tests/component.rs`). Phase 2's +runtime and service parts are in `crates/wasm-host/src/component/files.rs` and +`crates/shell/src/wasm_service.rs`, with the measurements below. Phase 3's are +`component/net.rs` (`wasi:http`), `component/host.rs` with +`wit/octosense-host.wit` (`octosense:host`), `Runtime::precompile` with +`wasm_service::warm` (compiling at install), and Pulley on OpenHarmony and in +tests; Windows runs the service, its tests in CI. App Hub's and App Flow's +parts are in review, and iOS and phase 4 are the plan. It extends [ADR 0011](0011-apps-own-functions-in-webassembly.md): core modules keep working as they do. How WebAssembly runs on `main`: [WebAssembly in OctoSense](../wasm.md). @@ -60,19 +66,41 @@ component reaches only what its app may already reach. They're listed as skipped, with the reason. 3. **Instances keep their state.** One instance per component lives for as long as the app's worker, so a component can hold a document, a cache or a model. - A trap or a deadline spends it; the next call gets a fresh one. An update, a - grant change or a withdrawal discards it, as for modules. + A worker holding one waits a minute for the next call, not five seconds. A + trap or a deadline spends the instance; the next call gets a fresh one. An + update, a grant change or a withdrawal discards it, as for modules. 4. **WASI scoped to the app's grants.** - **Always:** `wasi:clocks`, `wasi:random`, `wasi:io` and `wasi:cli`. stdout and stderr become the app's log lines (bounded). The environment, arguments and stdin are empty. - - **`wasi:filesystem`:** only with the app's `storage` capability. Its - storage folder is preopened as `/`, read-write, and nothing else of the - host's filesystem is visible. Without the capability there are no - preopens. - - **`wasi:http` outgoing, phase 3:** only to the hosts in the manifest's - `network.hosts`, through the shell's network policy. - - **Never:** `wasi:sockets`, and any import outside these WASI packages. A + - **`wasi:filesystem`:** only with the app's `storage` capability (and a + signed-in account, for an app with accounts), decided per call by the + rules an engine's folder follows. Its storage folder is preopened as `/`, + read-write, and nothing else of the host's filesystem is visible. Without + the capability there are no preopens. + - **The storage quota, per write.** A call may add what is left of the + app's quota when it starts. The runtime replaces the WASI calls that + grow a file (`write`, `set-size`, and the streams `write-via-stream` and + `append-via-stream` hand out) with ones that charge the growth, and the + calls that free bytes (`open-at` truncating, `unlink-file-at`) with ones + that give them back. Past the budget a write fails inside the component, + and the call's error says why. A check after each call instead was + rejected: a call could write gigabytes before it, and it costs two walks + of the folder a call. + - **`wasi:http` outgoing, phase 3:** to any host. The app declares `net`, + so that its install sheet says it uses the network, but neither that + nor `network.hosts` is enforced while it runs: the ruling of + 8 October 2026 removed the per-app runtime gates, and the OS and the + host's API surface are the boundary. Holding a component to its app's + `network.hosts`, and then to public HTTPS hosts, was rejected under that + ruling. A request waits outside the guest, where the epoch check cannot + end it, so its timeouts are clamped to the call's deadline, which is + 10 s for a component that imports `wasi:http`. + - **`octosense:host`, phase 3:** `request(service, args)` reaches the host + services the app is granted, dispatched on the UI thread as the app's + script's `host.request` is, but with no sheet and no prompt (only + methods a background surface may call), and never `wasm.*`. + - **Never:** `wasi:sockets`, and any import outside these packages. A component that asks for one is refused when it loads (`LoadError::Import`) and by App Hub's gate. 5. **No WIT to write.** @@ -93,10 +121,10 @@ component reaches only what its app may already reach. 6. **Admission sees what a component reaches.** App Hub's gate reads each component's imports: - it refuses any outside the allowed set; - - it requires `storage` for `wasi:filesystem` and `network.hosts` for - `wasi:http`; + - it requires `storage` for `wasi:filesystem` and `net` for `wasi:http`, + so that the install sheet shows them; - it tells reviewers what the component reaches, for example "files in its - app folder; HTTPS to api.example.com". + app folder; the network". Components need a new app contract version. ## Phases @@ -105,7 +133,7 @@ component reaches only what its app may already reach. | --- | --- | | 1. Runtime spike (done here) | `crates/wasm-host::component`: load, check imports, list exports with WIT signatures, long-lived instances, JSON calls, the WASI subset above with the storage preopen, the deadline, the memory cap and logs. Tests run an unmodified crate (`pulldown-cmark`) built with plain cargo, and refuse a component that imports `wasi:sockets`. | | 2. Usable by developers | The shell's `wasm` service: loading components from `fns/`, `wasm.` calls, per-app instances, the storage grant from the manifest, and storage quota accounting for component writes. A larger input limit for components. The guest SDK and its macro; `octo wasm new/build/doctor`; App Hub's gate check and contract version; docs and an example app. | -| 3. Reach and platforms | `wasi:http` outgoing under `network.hosts`; an `octosense:host` import for host services under the same checks as `host.request`; compiling at install time, so a phone skips the first compile; Pulley (Wasmtime's interpreter) on iOS and, until its JIT policy is known, OpenHarmony; Windows once its CI runs the runtime's tests. | +| 3. Reach and platforms | `wasi:http` outgoing to any host (declared with `net`, not enforced); an `octosense:host` import for host services under the same checks as `host.request`; compiling at install time, so a phone skips the first compile; Pulley (Wasmtime's interpreter) on iOS and, until its JIT policy is known, OpenHarmony; Windows once its CI runs the runtime's tests. | | 4. Shared components | Reviewed, versioned components in App Hub's catalog that apps depend on, like npm packages. The installer verifies them, and every app still gets its own instance and grants. | ## Alternatives considered @@ -132,14 +160,48 @@ component reaches only what its app may already reach. and randomness are new to Wasm but available to every script. - **Persistent instances use memory for longer.** The per-app worker cap (`MAX_WORKERS`) and the memory cap bound it, and idle workers exit as today. -- **Spike sizes:** `notes` (with `pulldown-cmark` and `getrandom`) is 310 KB; - the sockets probe is 123 KB. Timings are not measured yet; phase 2 records - them as ADR 0011 did. +- **Spike sizes:** `notes` (with `pulldown-cmark` and `getrandom`) is 313 KiB; + the sockets probe is 120 KiB. +- **A full disk looks like an I/O error.** wasi-libc reports any failed stream + write as `EIO`, whatever error the host gives, so most refused writes reach + the guest as an I/O error rather than `ENOSPC`. The runtime adds the reason + to the call's error and to the app's log. + +## Measurements + +`cargo run --release -p octosense-wasm-host --example measure_component`, on +an Apple M5 Max with macOS 26.6.2 (9 October 2026). Each figure is a median +of 200 calls unless the row says otherwise. The module is Wasm Lab's +(ADR 0011), which runs the same `pulldown-cmark`. + +| | Component | Module | Native | +| --- | --- | --- | --- | +| Compile with Cranelift, first load | 37.5 ms (313 KiB) | | | +| Load from the cache | 3.6 ms | | | +| Instantiate (with the storage folder) | 0.09 ms (0.10 ms) | | | +| Markdown to HTML, 32 KiB | 468 µs | 400 µs | 183 µs | +| Render and write 49 KiB to a new file | 678 µs | | 156 µs (the write) | +| Write a 12-byte file | 143 µs | | | +| Read 49 KiB back | 59 µs | | | +| 1 MiB of `list`, there and back as base64 | 18.4 ms | | | +| The smallest call | 0.2 µs | 0.3 µs | | + +In Pulley (`OCTOSENSE_WASM_PULLEY=1`, `--features pulley`, same machine), the +Markdown function takes 15.1 ms as a component and 14.1 ms as a module: about +32 times Cranelift's code. The smallest component call is 0.4 µs. OpenHarmony +runs this until its code-generation policy is known. + +The quota needs what the folder holds when a call starts: one walk of it, as +for an engine's call. That took 0.12 ms for 10 files, 2.0 ms for 1,000 and +23.8 ms for 10,000 (a median of 50). Rewriting one file over and over is +slow on APFS, natively too (a median of about 6 ms a rewrite), so the +example writes a new file each time. ## Open questions -- Quota for component writes: account each write (by wrapping the filesystem - host) or check after each call. Phase 2 decides by measuring. -- Whether a component may call host services that open sheets (sign-in, - review). Probably only through the script, never from a component's call. +Settled in phase 3: a component never calls a host service that opens a +sheet or asks the person; such a call belongs to the script. + - Async functions (WASI 0.3) and streaming bodies: after phase 3. +- iOS: a Home build with the service in Pulley; the Rust target is not + installed on the build machine yet. diff --git a/docs/adr/0014-app-components-in-webassembly.zh-CN.md b/docs/adr/0014-app-components-in-webassembly.zh-CN.md index 53f22a80d..b4153da4e 100644 --- a/docs/adr/0014-app-components-in-webassembly.zh-CN.md +++ b/docs/adr/0014-app-components-in-webassembly.zh-CN.md @@ -3,7 +3,12 @@ [English](0014-app-components-in-webassembly.md) | 简体中文 状态：提议（2026 年 10 月 9 日）。第 1 阶段（运行时验证原型）位于 -`crates/wasm-host`（`src/component.rs`、`tests/component.rs`）；第 2–4 阶段是下文的计划。 +`crates/wasm-host`（`src/component.rs`、`tests/component.rs`）。第 2 阶段的运行时和服务 +部分位于 `crates/wasm-host/src/component/files.rs` 和 `crates/shell/src/wasm_service.rs`， +测量结果见下文。第 3 阶段的部分是 `component/net.rs`（`wasi:http`）、`component/host.rs` 与 +`wit/octosense-host.wit`（`octosense:host`）、`Runtime::precompile` 与 `wasm_service::warm` +（安装时编译），以及 OpenHarmony 上和测试中的 Pulley；Windows 运行这个服务，其测试在 CI 中运行。 +App Hub 与 App Flow 的部分正在审阅，iOS 和第 4 阶段是计划。 本 ADR 扩展 [ADR 0011](0011-apps-own-functions-in-webassembly.zh-CN.md)：核心模块照旧可用。 `main` 上 WebAssembly 的运行方式见 [OctoSense 中的 WebAssembly](../wasm.zh-CN.md)。 @@ -46,15 +51,28 @@ crates.io 上的大多数 crate 要么需要上述缺失能力中的某些（时 - 函数返回 `result` 时，结果为其值，或以 `E` 作为调用的错误。 - 接收或返回资源（resource）的函数不能从脚本调用，会被列为已跳过并附原因。 3. **实例保留状态。** 每个组件一个实例，在应用的 worker 存活期间一直存在，因此组件可以保存文档、 - 缓存或模型。陷阱（trap）或超时会使实例作废，下一次调用会得到新实例。应用更新、授权变更或撤回 - 时实例会被丢弃，与模块相同。 + 缓存或模型。持有实例的 worker 等待下一次调用一分钟，而不是五秒。陷阱（trap）或超时会使实例 + 作废，下一次调用会得到新实例。应用更新、授权变更或撤回时实例会被丢弃，与模块相同。 4. **WASI 的范围与应用授权一致。** - **始终提供：** `wasi:clocks`、`wasi:random`、`wasi:io` 和 `wasi:cli`。stdout 与 stderr 成为应用的日志行（有上限）。环境变量、参数和 stdin 都为空。 - - **`wasi:filesystem`：** 仅在应用有 `storage` 能力时提供。其存储文件夹以读写方式预打开为 `/`， + - **`wasi:filesystem`：** 仅在应用有 `storage` 能力（有账户的应用还需已登录账户）时提供， + 每次调用都按引擎文件夹所遵循的规则决定。其存储文件夹以读写方式预打开为 `/`， 主机文件系统的其余部分一概不可见。没有该能力时不预打开任何目录。 - - **`wasi:http` 出站（第 3 阶段）：** 只能访问清单 `network.hosts` 中的主机，并经过 shell 的网络策略。 - - **从不提供：** `wasi:sockets`，以及上述 WASI 包以外的任何导入。请求这些导入的组件在加载时被拒绝 + - **存储配额逐次写入计量。** 一次调用可以写入的量，是调用开始时应用配额的剩余部分。运行时把使文件 + 变大的 WASI 调用（`write`、`set-size`，以及 `write-via-stream` 与 `append-via-stream` 返回 + 的流）替换为计量增长的版本，把释放字节的调用（截断的 `open-at`、`unlink-file-at`）替换为 + 归还字节的版本。超出预算的写入在组件内部失败，调用的错误会说明原因。我们没有采用"每次调用后 + 检查"：一次调用在检查之前就可能写入数 GB，而且每次调用要遍历文件夹两次。 + - **`wasi:http` 出站（第 3 阶段）：** 可以访问任何主机。应用声明 `net`，让安装界面说明它会使用 + 网络，但运行时既不强制 `net`，也不强制 `network.hosts`：2026 年 10 月 8 日的裁定取消了按应用的 + 运行时闸门，边界是操作系统和宿主的 API 表面。把组件限制在应用的 `network.hosts` 内、后来又限制 + 为公共 HTTPS 主机的做法，都已按该裁定否决。请求在客体之外等待，epoch 检查无法结束它，因此 + 请求的超时被限制在调用的截止时间内；导入 `wasi:http` 的组件每次调用有 10 秒。 + - **`octosense:host`（第 3 阶段）：** `request(service, args)` 可以调用应用已获授权的宿主服务， + 像应用脚本的 `host.request` 一样在 UI 线程上分派，但不打开面板、不询问用户（只能调用后台界面 + 可以调用的方法），并且绝不调用 `wasm.*`。 + - **从不提供：** `wasi:sockets`，以及上述包以外的任何导入。请求这些导入的组件在加载时被拒绝 （`LoadError::Import`），App Hub 的审核闸门也会拒绝。 5. **无需编写 WIT。** - **Guest SDK（第 2 阶段）：** guest crate `octosense-component` 提供 @@ -68,8 +86,8 @@ crates.io 上的大多数 crate 要么需要上述缺失能力中的某些（时 - **分发：** SDK 发布到 crates.io（需维护者批准）；发布前使用 git 依赖。 6. **审核能看到组件的访问范围。** App Hub 的闸门读取每个组件的导入： - 拒绝允许集合以外的导入； - - `wasi:filesystem` 需要 `storage` 能力，`wasi:http` 需要 `network.hosts`； - - 告诉审核者组件能访问什么，例如“其应用文件夹中的文件；访问 api.example.com 的 HTTPS”。 + - `wasi:filesystem` 需要 `storage` 能力，`wasi:http` 需要 `net` 能力，以便安装界面展示它们； + - 告诉审核者组件能访问什么，例如“其应用文件夹中的文件；网络”。 组件需要新的应用合约版本。 ## 阶段 @@ -78,7 +96,7 @@ crates.io 上的大多数 crate 要么需要上述缺失能力中的某些（时 | --- | --- | | 1. 运行时验证原型（本 PR） | `crates/wasm-host::component`：加载、检查导入、列出导出及其 WIT 签名、长期存活的实例、JSON 调用、上述 WASI 子集与存储预打开、超时、内存上限和日志。测试运行一个用普通 cargo 构建的、未作修改的 crate（`pulldown-cmark`），并拒绝导入 `wasi:sockets` 的组件。 | | 2. 开发者可用 | shell 的 `wasm` 服务：从 `fns/` 加载组件、`wasm.` 调用、按应用的实例、来自清单的存储授权，以及组件写入的存储配额计量。提高组件的输入上限。guest SDK 及其宏；`octo wasm new/build/doctor`；App Hub 闸门检查与合约版本；文档与示例应用。 | -| 3. 访问能力与平台 | 受 `network.hosts` 约束的 `wasi:http` 出站；`octosense:host` 导入，以与 `host.request` 相同的检查调用主机服务；安装时编译，让手机跳过首次编译；iOS 使用 Pulley（Wasmtime 的解释器），OpenHarmony 在其 JIT 策略明确前也使用 Pulley；Windows 待其 CI 运行运行时测试后开启。 | +| 3. 访问能力与平台 | 可访问任何主机的 `wasi:http` 出站（以 `net` 声明，不在运行时强制）；`octosense:host` 导入，以与 `host.request` 相同的检查调用主机服务；安装时编译，让手机跳过首次编译；iOS 使用 Pulley（Wasmtime 的解释器），OpenHarmony 在其 JIT 策略明确前也使用 Pulley；Windows 待其 CI 运行运行时测试后开启。 | | 4. 共享组件 | App Hub 目录中经过审核、带版本的组件，应用可以像 npm 包一样依赖它们。安装器负责校验，每个应用仍有自己的实例和授权。 | ## 考虑过的替代方案 @@ -99,11 +117,41 @@ crates.io 上的大多数 crate 要么需要上述缺失能力中的某些（时 时钟和随机数对 Wasm 是新增的，但每个脚本本来就能使用。 - **常驻实例会更长时间占用内存。** 每应用 worker 上限（`MAX_WORKERS`）与内存上限约束其用量， 空闲的 worker 照旧退出。 -- **验证原型的大小：** `notes`（含 `pulldown-cmark` 与 `getrandom`）为 310 KB，sockets 探测组件为 123 KB。 - 耗时尚未测量；第 2 阶段会像 ADR 0011 一样记录。 +- **验证原型的大小：** `notes`（含 `pulldown-cmark` 与 `getrandom`）为 313 KiB，sockets 探测组件为 120 KiB。 +- **磁盘已满看起来像 I/O 错误。** 无论宿主给出什么错误，wasi-libc 都把失败的流写入报告为 `EIO`， + 因此大多数被拒绝的写入在客体中表现为 I/O 错误，而不是 `ENOSPC`。运行时会把原因加到调用的错误 + 和应用的日志中。 + +## 测量 + +在 Apple M5 Max、macOS 26.6.2 上运行 +`cargo run --release -p octosense-wasm-host --example measure_component`（2026 年 10 月 9 日）。 +除非该行另有说明，每个数字都是 200 次调用的中位数。模块是 Wasm Lab 的（ADR 0011），运行同一个 +`pulldown-cmark`。 + +| | 组件 | 模块 | 原生 | +| --- | --- | --- | --- | +| Cranelift 编译（首次加载） | 37.5 毫秒（313 KiB） | | | +| 从缓存加载 | 3.6 毫秒 | | | +| 实例化（带存储文件夹） | 0.09 毫秒（0.10 毫秒） | | | +| Markdown 转 HTML，32 KiB | 468 微秒 | 400 微秒 | 183 微秒 | +| 渲染并写入一个 49 KiB 的新文件 | 678 微秒 | | 156 微秒（仅写入） | +| 写入一个 12 字节的文件 | 143 微秒 | | | +| 读回 49 KiB | 59 微秒 | | | +| 1 MiB 的 `list`，以 base64 往返 | 18.4 毫秒 | | | +| 最小的调用 | 0.2 微秒 | 0.3 微秒 | | + +在 Pulley 中（`OCTOSENSE_WASM_PULLEY=1`、`--features pulley`，同一台机器），Markdown 函数作为组件 +耗时 15.1 毫秒，作为模块耗时 14.1 毫秒，约为 Cranelift 代码的 32 倍；最小的组件调用为 0.4 微秒。 +在 OpenHarmony 的代码生成策略明确之前，OpenHarmony 以这种方式运行。 + +计量配额需要知道调用开始时文件夹里有多少数据：遍历一次文件夹，与引擎的调用相同。10 个文件耗时 +0.12 毫秒，1,000 个 2.0 毫秒，10,000 个 23.8 毫秒（50 次的中位数）。在 APFS 上反复改写同一个 +文件很慢，原生代码也一样（每次改写的中位数约 6 毫秒），因此示例每次都写入新文件。 ## 待定问题 -- 组件写入的配额：逐次计量（包装文件系统宿主实现），还是在每次调用后检查。第 2 阶段通过测量决定。 -- 组件能否调用会打开面板（登录、审阅）的主机服务。很可能只允许经由脚本，绝不允许在组件的调用中进行。 +第 3 阶段已定：组件绝不调用会打开面板或询问用户的宿主服务，这类调用属于脚本。 + - 异步函数（WASI 0.3）和流式内容：第 3 阶段之后。 +- iOS：在 Pulley 中带这个服务构建 Home；构建机器上还没有安装 Rust 的 iOS 目标。 diff --git a/docs/wasm.md b/docs/wasm.md index f5f251c51..42fb9d5a3 100644 --- a/docs/wasm.md +++ b/docs/wasm.md @@ -13,7 +13,7 @@ have it and how it was checked. It describes `main` after #400 (8 October | Where | Whose module | Runtime | Builds | State | | --- | --- | --- | --- | --- | -| [An app's own functions](#an-apps-own-functions-the-wasm-service): the `wasm` service ([ADR 0011](adr/0011-apps-own-functions-in-webassembly.md)) | The app's own bundle, `fns/*.wasm`, under the `wasm` capability | Wasmtime 49, compiled by Cranelift (`crates/wasm-host`) | Every standard desktop and Home build on macOS, Linux and Android (feature `wasm-functions`); none for Windows, iOS or OpenHarmony | Accepted, limited support. Tests pass on macOS and Linux; a OnePlus 6 passed the phone acceptance. No release has it yet | +| [An app's own functions](#an-apps-own-functions-the-wasm-service): the `wasm` service ([ADR 0011](adr/0011-apps-own-functions-in-webassembly.md)) | The app's own bundle, `fns/*.wasm`, under the `wasm` capability | Wasmtime 49, compiled by Cranelift (`crates/wasm-host`); components get WASI 0.2 from `wasmtime-wasi` | Every standard desktop and Home build on macOS, Windows, Linux, Android and OpenHarmony (feature `wasm-functions`; on OpenHarmony in Pulley, Wasmtime's interpreter); none for iOS | Accepted, limited support. Tests pass on macOS and Linux; a OnePlus 6 passed the phone acceptance. No release has it yet | | [Engine plug-ins](#engine-plug-ins-the-photo-and-vector-services): photocraft behind `photo`, vectorcraft behind `vector` ([ADR 0013](adr/0013-craft-engines-as-pinned-services.md)) | None: both services refuse every `plugin.*` command (#398, #405) | wasmi 2, an interpreter | Linked into every build with App Hub | Closed: no caller can install or run a plug-in | | [Splash's math compiler](#splash-math-and-compute-kernels) (makepad) | Generated by makepad from Splash code | makepad-stitch, an interpreter | Linked into every build | Not used: OctoSense links nothing that calls it | | [The shell in a browser](#the-shell-in-a-browser) | The shell itself, built for `wasm32-unknown-unknown` | The browser | None | Does not build | @@ -28,8 +28,9 @@ conversion. Splash's compute kernels stay the answer for numeric work. ### The module -A module is core WebAssembly at `fns/.wasm`, at most 8 per bundle. There -is no component model and no WASI. +A module is core WebAssembly at `fns/.wasm`, at most 8 per bundle, with +no WASI. A file there may instead be a [component](#components), which has +both. - It exports `memory`, `octo_alloc(len) -> ptr` and `octo_free(ptr, len)`. - Every other export of type `(i32, i32) -> i64` is a function: it takes its @@ -44,11 +45,68 @@ The guest crate [`apps/wasmlab/guest/octosense-guest`](../apps/wasmlab/guest/octosense-guest/src/lib.rs) writes all of this: `export!` for `fn(&[u8]) -> Result, String>`, `export_json!` for serde types, and `log`. Build the crate as a `cdylib` for -`wasm32-unknown-unknown`. Builds for `wasm32-wasip1`, wasm-bindgen output and -components import functions the shell does not provide, so they do not load. +`wasm32-unknown-unknown`. Builds for `wasm32-wasip1` and wasm-bindgen output +import functions the shell does not provide, so they do not load; a build for +`wasm32-wasip2` is a [component](#components), the other kind it runs. App developers start from App Flow's [Run your own Rust code](https://github.com/OctoSense-org/OctoSense-App-Flow/blob/main/docs/RUST.md). +### Components + +[ADR 0014](adr/0014-app-components-in-webassembly.md) (Proposed) adds a second +kind of `fns/.wasm`: a WebAssembly component, built from an ordinary +Rust crate with `cargo build --target wasm32-wasip2`. The service tells the +two kinds apart by the file's preamble, and a bundle may carry both. + +- **No glue.** Every function the component exports is `wasm.`, + named in snake_case: `save-html` is `wasm.save_html`, and either spelling + is accepted. Arguments are an object keyed by parameter name, an array in + parameter order or, for one parameter, the value itself. A record comes + back as an object, an enum as its name, `list` as base64 and a + `result` as its value or the request's error. `wasm.functions` lists + every export with its WIT signature; a function that takes or returns a + resource is listed as skipped, with the reason. +- **State.** Each component's instance lives on in the app's worker between + calls, so it can keep a parsed document or a cache. A trap or the deadline + spends it and the next call gets a new one; an update, a changed grant, a + withdrawal or the worker's exit (a minute without calls) ends it. +- **What it reaches.** Clocks and random numbers. Its stdout and stderr become + the app's log lines, and it gets no environment, arguments or stdin. A + component that imports anything beyond `wasi:cli`, `wasi:clocks`, + `wasi:filesystem`, `wasi:http`, `wasi:io`, `wasi:random` and + `octosense:host`, such as sockets, is refused when it loads. +- **Files.** With the `storage` capability (and a signed-in account, for an + app with accounts), the app's own storage folder, the one its script's + `fs.*` sees, is the component's `/`. Nothing else on the device is + visible, and without the capability there is no filesystem. +- **The storage quota.** A call may add what is left of the app's quota when + the call starts. A write past it fails inside the component: `ftruncate` + reports a full disk, and an ordinary write an I/O error, because wasi-libc + reports any failed stream write as one. The request's error then ends with + "a write was refused: the storage budget is used up". Rewriting, truncating + and deleting give bytes back, so a component can free space. +- **The network.** Its `wasi:http` requests reach any host, as the device + can: an app's network declarations (`net`, `network.hosts`) are shown when + it is installed and not enforced while it runs. The ruling of 8 October + 2026 removed the per-app runtime gates; the OS and the host's API surface + are the boundary. (The app's Splash script still passes the Splash + runtime's URL gate until the Makepad fork drops it.) A component that + imports `wasi:http` gets 10 s a call instead of 2 s, and each request's + connect, first-byte and between-bytes timeouts end with the call. +- **Host services.** Through `octosense:host` + ([`crates/wasm-host/wit/octosense-host.wit`](../crates/wasm-host/wit/octosense-host.wit)), + `request(service, args)` calls one of the host services its app is + granted, as the app's script does: the same families, dispatched on the UI + thread, but never with a sheet or a prompt, so only methods a background + surface may call. `wasm.*` is refused, since the app's worker is busy with + the call, and the call's deadline bounds the wait. + +ADR 0014 records what this costs. On an M-series Mac, the 313 KiB test +component compiles in about 40 ms and loads from the cache in about 4 ms; +its Markdown function runs about 2.6 times slower than native Rust, close to +the same code as a module. In Pulley it runs about 32 times slower than +that. + ### How a module reaches a device - **A store app.** App Hub's gate (app contract 1.7.0) admits a `.wasm` file @@ -76,10 +134,11 @@ former name). Builds for Windows, iOS and OpenHarmony leave the runtime out - `wasm.functions`: the functions the app's modules export, how each module loaded (compiled, or from the cache, and how long it took) and how each function has run. -- `wasm.`: one call. A string argument goes in as its text, anything - else as JSON. JSON output comes back as data, anything else as - `{"text": …}`. The function's own error, a trap or the deadline is the - request's error. +- `wasm.`: one call. For a module, a string argument goes in as its + text and anything else as JSON; JSON output comes back as data, anything + else as `{"text": …}`. A component's arguments and result are typed + ([Components](#components)). The function's own error, a trap or the + deadline is the request's error. The code always comes from the calling app's own admitted, digest-checked bundle, never from an argument, and no app reaches another app's functions. @@ -87,9 +146,10 @@ The Card runner's gate and the tool executor require the `wasm` capability, and the service checks the admitted manifest's grant again before it loads anything. -Every call gets a fresh instance: memory, globals, tables and log lines from -one call never reach the next, including after a success or the function's -own error. Only compiled code is reused, in the app's worker and in a disk +Every call to a module gets a fresh instance: memory, globals, tables and log +lines from one call never reach the next, including after a success or the +function's own error. A component keeps one instance between calls instead +([Components](#components)). Only compiled code is reused, in the app's worker and in a disk cache (`wasm-cache` in the host directory, keyed by the module's SHA-256 and the engine's compatibility hash). The service checks admission again before a call runs and before its answer is delivered. An update, a changed grant or a @@ -103,24 +163,30 @@ be interrupted, but an expired or cancelled request never goes on to run. | Table elements | 16,384 | | Wasm stack | 512 KiB | | Module | 8 MiB | -| Input or output of one call | 16 MiB in the runtime; 1 MiB of serialized input in the service | +| Input or output of one call | 16 MiB in the runtime; 8 MiB of serialized input in the service | | Log | 64 lines a call, 1 KiB each | | Modules per app | 8 | | Apps running functions at once | 4 workers, one per app | | Queued requests | 4 per app | -| Buffered input, all apps | 16 MiB | +| Buffered input, all apps | 32 MiB | | One request, queueing and loading included | 10 s | -| An idle worker | exits after 5 s | +| An idle worker | exits after 5 s, or after 60 s while it holds a component's instance | A full queue or no free worker fails the request at once instead of making it wait. +When App Hub installs or updates an app, the shell compiles its functions into +the cache in the background (`wasm_service::warm`): one app at a time, on a +thread of its own, from the admitted bundle and only with the `wasm` grant. The +app's first call then loads from the cache instead of waiting for Cranelift. + ### Agent tools An app's `tools.json` may map a tool to one of its functions with `"host_method": "wasm."`. That needs the `wasm` capability. Unlike shared host methods it has no minimum risk and needs no -`"private_data": true`, because the function sees only the arguments it is -given. Wasm Lab maps `wasmlab.find_slots`, `wasmlab.rank` and `wasmlab.diff` +`"private_data": true`: a module sees only the arguments it is given, and a +component also its own app's folder, as the app's script does. Neither +reaches shared data. Wasm Lab maps `wasmlab.find_slots`, `wasmlab.rank` and `wasmlab.diff` to `wasm.find_slots`, `wasm.fuzzy_rank` and `wasm.text_diff`, all `read`. ### Platforms @@ -130,9 +196,9 @@ to `wasm.find_slots`, `wasm.fuzzy_rank` and `wasm.text_diff`, all `read`. | macOS | Runs. Validated in a hidden desktop shell (ADR 0011). Wasmtime catches traps with Mach exception ports, not signals. | | Linux | Runs. The runtime and service tests pass on x86_64, with traps caught by signals, and Wasm Lab ran in a headless desktop (8 October 2026). | | Android | Runs in Home's default build. Needs libc 0.2.190 or later, which `crates/wasm-host` requires: with 0.2.189 every trap ended the process. Validated on a Redmi Note 12 (ADR 0011) and by the [phone acceptance](#phone-acceptance) on a OnePlus 6. The runtime adds about 7.5 MiB of code; compiling a 433 KiB module takes about 0.4 s on a Snapdragon 685, and loading it from the cache 5–11 ms. | -| Windows | Not included. The runtime stays out until it has been checked on Windows. | -| iOS | Not included. Apps may not JIT there; the option is Pulley, Wasmtime's interpreter, about 17 times slower than Cranelift. | -| OpenHarmony | Not included. Its JIT policy is **unverified**. | +| Windows | Included (ADR 0014). [`wasm-windows.yml`](../.github/workflows/wasm-windows.yml) runs the runtime's tests and compiles the shell with the service on GitHub's `windows-2022`. A desktop run on Windows is **unverified**. | +| iOS | Not included. Apps may not generate code there, so the runtime would use Pulley, as on OpenHarmony; no iOS build with the service has been compiled yet. | +| OpenHarmony | Included, in Pulley, Wasmtime's interpreter, until OpenHarmony's code-generation policy is known: Cranelift compiles to Pulley's bytecode and nothing runs as native code. Home's OpenHarmony release build compiled with the service (9 October 2026, `cargo-makepad makepad ohos … deveco -p octosense-home --release`); no device has run it (**unverified**). Pulley is about 32 times slower than Cranelift (ADR 0014). | On a managed Mac, Microsoft Defender holds the first open of a freshly written cache file for about a second; ADR 0011 has the measurements. @@ -186,10 +252,36 @@ cd phone && cargo test --locked --features mobile-apps -p octosense-shell wasm_s its concurrent writers, cancellation. - `crates/wasm-host/tests/guest.rs` runs Wasm Lab's module and compares every function with the same Rust run natively, then every `rogue` mode. +- `crates/wasm-host/tests/component.rs` runs four components built from + `crates/wasm-host/tests/component-guest`: an unmodified crate + (pulldown-cmark) called with JSON, records and snake_case names, state + kept on one instance, clocks, randomness and bytes, files only in the + granted folder (also read-only), the storage budget, the deadline and the + memory cap; `fetch`, whose `wasi:http` reaches a local server only under a + granted host, and whose request to a server that never answers ends at the + deadline; `hostcall`, which calls host services through the embedder; and + a component that imports sockets refused. - The service tests cover the tool mapping, bounded input, a fresh instance per call, revocation after an update, a grant change or a withdrawal, cancellation and the queue bounds, and the app's own functions answering - its tools and its script. + its tools and its script. For a component they cover typed calls, its + instance's state, the app's storage folder and its quota, an app without + the storage capability, a component shipped as a system app through App + Hub's admission, requests that reach only the app's hosts, host calls that + reach only its granted services, and an installed app's first call + loading from the cache. +- `cargo run --release -p octosense-wasm-host --example measure_component` + prints a component's timings against a module and native Rust. +- `OCTOSENSE_WASM_PULLEY=1 cargo test --locked -p octosense-wasm-host --features pulley` + runs the same tests in Pulley, as OpenHarmony runs them; `phone.yml` runs + it too. + +The test components rebuild byte for byte: with rustc 1.97.1, +`crates/wasm-host/tests/component-guest/build.sh` writes the same +`notes.component.wasm` (SHA-256 `963c965b…`), `netprobe.component.wasm` +(`98b2402a…`), `fetch.component.wasm` (`6eaf6ac8…`) and +`hostcall.component.wasm` (`d39f2a8e…`). It needs the target once: +`rustup target add wasm32-wasip2`. `phone.yml` runs the first command, and the service's tests as part of Home's (`mobile-apps` includes `wasm-functions`); `tools/ci-local.sh --linux-host @@ -217,14 +309,15 @@ command. - No release includes the service yet: the first desktop and Home releases built from `main` after #400 will. -- Windows, iOS (Pulley) and OpenHarmony. +- iOS: building Home for iOS with the service (in Pulley). - CPU fairness between apps, and a bound on the disk cache, which nothing evicts. -- Compiling at install time (store apps) or build time (system apps), to - remove the first compile on a phone. -- Typed interfaces (the component model and WIT) instead of JSON. Any import - beyond `octo.log`, such as a clock or randomness, would be a new - capability. +- Compiling system apps' functions ahead: an installed app's are compiled at + install, a system app's at its first call. +- Components on a phone: Android and OpenHarmony builds link the runtime, + but no component has run on a device yet (**unverified**). +- Bytes are slow through JSON: a 1 MiB `list` costs about 18 ms there + and back, as base64. ## Engine plug-ins: the `photo` and `vector` services diff --git a/docs/wasm.zh-CN.md b/docs/wasm.zh-CN.md index 1626b3b08..9ddffeff7 100644 --- a/docs/wasm.zh-CN.md +++ b/docs/wasm.zh-CN.md @@ -10,7 +10,7 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 | 位置 | 谁的模块 | 运行时 | 构建 | 状态 | | --- | --- | --- | --- | --- | -| 应用自带的函数：`wasm` 服务（[ADR 0011](adr/0011-apps-own-functions-in-webassembly.zh-CN.md)） | 应用自己的应用包，`fns/*.wasm`，需要 `wasm` 能力 | Wasmtime 49，由 Cranelift 编译（`crates/wasm-host`） | macOS、Linux 和 Android 上的每个标准桌面版和 Home 构建（特性 `wasm-functions`）；Windows、iOS 和 OpenHarmony 不包含 | 已接受，有限支持。测试在 macOS 和 Linux 上通过；一部 OnePlus 6 通过了手机验收。尚无发布版本包含它 | +| 应用自带的函数：`wasm` 服务（[ADR 0011](adr/0011-apps-own-functions-in-webassembly.zh-CN.md)） | 应用自己的应用包，`fns/*.wasm`，需要 `wasm` 能力 | Wasmtime 49，由 Cranelift 编译（`crates/wasm-host`）；组件的 WASI 0.2 来自 `wasmtime-wasi` | macOS、Windows、Linux、Android 和 OpenHarmony 上的每个标准桌面版和 Home 构建（特性 `wasm-functions`；OpenHarmony 上在 Wasmtime 的解释器 Pulley 中运行）；iOS 不包含 | 已接受，有限支持。测试在 macOS 和 Linux 上通过；一部 OnePlus 6 通过了手机验收。尚无发布版本包含它 | | 引擎插件：`photo` 背后的 photocraft、`vector` 背后的 vectorcraft（[ADR 0013](adr/0013-craft-engines-as-pinned-services.zh-CN.md)） | 无：两个服务都拒绝所有 `plugin.*` 命令（#398、#405） | wasmi 2，解释器 | 链接进所有带 App Hub 的构建 | 已关闭：任何调用方都不能安装或运行插件 | | Splash 的数学编译器（makepad） | makepad 根据 Splash 代码生成 | makepad-stitch，解释器 | 链接进所有构建 | 未使用：OctoSense 没有链接任何调用它的代码 | | 浏览器中的外壳 | 外壳本身，为 `wasm32-unknown-unknown` 构建 | 浏览器 | 无 | 无法构建 | @@ -23,8 +23,8 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 ### 模块 -模块是放在 `fns/.wasm` 的核心 WebAssembly，每个应用包最多 8 个。不使用组件模型， -也不使用 WASI。 +模块是放在 `fns/.wasm` 的核心 WebAssembly，每个应用包最多 8 个，不使用 WASI。 +那里的文件也可以是[组件](#组件)，组件两者都有。 - 它导出 `memory`、`octo_alloc(len) -> ptr` 和 `octo_free(ptr, len)`。 - 其余类型为 `(i32, i32) -> i64` 的导出都是函数：在 `(ptr, len)` 处取输入，返回 @@ -37,11 +37,50 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 [`apps/wasmlab/guest/octosense-guest`](../apps/wasmlab/guest/octosense-guest/src/lib.rs) 生成这一切：`export!` 用于 `fn(&[u8]) -> Result, String>`，`export_json!` 用于 serde 类型，另有 `log`。把 crate 构建成面向 `wasm32-unknown-unknown` 的 -`cdylib`。面向 `wasm32-wasip1` 的构建、wasm-bindgen 的输出以及组件都会导入外壳不提供的 -函数，因此无法加载。应用开发者请从 App Flow 的 +`cdylib`。面向 `wasm32-wasip1` 的构建和 wasm-bindgen 的输出会导入外壳不提供的函数， +因此无法加载；面向 `wasm32-wasip2` 的构建是[组件](#组件)，即服务运行的另一种文件。应用开发者请从 App Flow 的 [Run your own Rust code](https://github.com/OctoSense-org/OctoSense-App-Flow/blob/main/docs/RUST.md) 开始。 +### 组件 + +[ADR 0014](adr/0014-app-components-in-webassembly.zh-CN.md)（提议）增加了第二种 +`fns/.wasm`：WebAssembly 组件，用 `cargo build --target wasm32-wasip2` 从普通的 +Rust crate 构建。服务按文件头区分两者，一个应用包可以同时携带两种。 + +- **无需胶水代码。** 组件导出的每个函数都是 `wasm.`，名称用 snake_case： + `save-html` 即 `wasm.save_html`，两种写法都接受。参数可以是按参数名作键的对象、按参数 + 顺序排列的数组；只有一个参数时也可以直接传值。记录返回为对象，枚举返回为其名称， + `list` 返回为 base64，`result` 返回为其值，或作为这次请求的错误。 + `wasm.functions` 列出每个导出及其 WIT 签名；接收或返回资源的函数会被列为已跳过，并附原因。 +- **状态。** 每个组件的实例在调用之间留在应用的工作线程里，因此可以保存已解析的文档或缓存。 + 陷阱或超时会使实例作废，下一次调用得到新实例；更新、授权变化、撤回或工作线程退出（一分钟 + 没有调用）都会结束它。 +- **能接触什么。** 时钟和随机数。它的 stdout 和 stderr 成为应用的日志行，没有环境变量、 + 参数和 stdin。导入 `wasi:cli`、`wasi:clocks`、`wasi:filesystem`、`wasi:http`、`wasi:io`、 + `wasi:random` 和 `octosense:host` 以外任何东西（例如套接字）的组件在加载时被拒绝。 +- **文件。** 应用有 `storage` 能力（有账户的应用还需已登录账户）时，应用自己的存储文件夹， + 即其脚本的 `fs.*` 看到的那个，就是组件的 `/`。设备上的其他东西一概不可见；没有这个能力 + 时没有文件系统。 +- **存储配额。** 一次调用可以写入的量，是调用开始时应用配额的剩余部分。超出的写入会在组件 + 内部失败：`ftruncate` 报告磁盘已满，普通写入报告 I/O 错误，因为 wasi-libc 把任何失败的 + 流写入都报告为 I/O 错误。这时请求的错误以 "a write was refused: the storage budget is + used up" 结尾。改写、截断和删除会归还字节，因此组件可以自己腾出空间。 +- **网络。** 组件的 `wasi:http` 请求可以访问任何主机，与设备本身能访问的范围相同：应用的网络 + 声明（`net`、`network.hosts`）在安装时展示，运行时不强制。2026 年 10 月 8 日的裁定取消了按应用 + 的运行时闸门，边界是操作系统和宿主的 API 表面。（在 Makepad 分支去掉之前，应用的 Splash 脚本 + 仍要经过 Splash 运行时的 URL 闸门。）导入 `wasi:http` 的组件每次调用有 10 秒，而不是 2 秒； + 每个请求的连接、首字节和字节间超时都随调用结束。 +- **宿主服务。** 通过 `octosense:host` + （[`crates/wasm-host/wit/octosense-host.wit`](../crates/wasm-host/wit/octosense-host.wit)）， + 组件可以用 `request(service, args)` 调用应用已获授权的宿主服务，与应用的脚本相同：同样的服务族， + 在 UI 线程上分派，但绝不打开面板、不询问用户，因此只能调用后台界面可以调用的方法。`wasm.*` + 被拒绝，因为应用的工作线程正忙于这次调用；调用的截止时间限制等待时长。 + +ADR 0014 记录了开销。在 M 系列 Mac 上，313 KiB 的测试组件约 40 毫秒完成编译，从缓存加载 +约 4 毫秒；它的 Markdown 函数比原生 Rust 慢约 2.6 倍，与同一段代码作为模块时接近。在 Pulley +中还要再慢约 32 倍。 + ### 模块如何到达设备 - **商店应用。** App Hub 的审核（应用契约 1.7.0）只接受放在 `fns/.wasm` 的 @@ -65,16 +104,17 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 - `wasm.functions`：应用的模块导出了哪些函数，每个模块如何加载（编译，还是取自缓存， 用了多久），以及每个函数的运行情况。 -- `wasm.`：一次调用。字符串参数按原文传入，其他参数按 JSON 传入。JSON 输出 - 作为数据返回，其他输出返回为 `{"text": …}`。函数自己返回的错误、陷阱或超时都是这次 - 请求的错误。 +- `wasm.`：一次调用。对模块而言，字符串参数按原文传入，其他参数按 JSON 传入； + JSON 输出作为数据返回，其他输出返回为 `{"text": …}`。组件的参数和结果带类型 + （见[组件](#组件)）。函数自己返回的错误、陷阱或超时都是这次请求的错误。 代码总是来自发起调用的应用自己的、已准入且摘要校验过的应用包，绝不来自参数；任何应用都 接触不到别的应用的函数。Card runner 的审核和工具执行器都要求 `wasm` 能力，服务在加载任何 东西之前还会再次检查已准入清单中的授权。 -每次调用都使用全新的实例：一次调用的内存、全局变量、表和日志行都不会进入下一次调用，成功 -调用或函数自己返回错误之后也一样。只有编译后的代码会被复用：保存在应用的工作线程里，以及 +每次调用模块都使用全新的实例：一次调用的内存、全局变量、表和日志行都不会进入下一次调用， +成功调用或函数自己返回错误之后也一样。组件则在调用之间保留一个实例（见[组件](#组件)）。 +只有编译后的代码会被复用：保存在应用的工作线程里，以及 磁盘缓存中（宿主目录下的 `wasm-cache`，以模块的 SHA-256 和引擎兼容性哈希为键）。调用 运行之前、答复交付之前，服务都会再次检查准入。更新、授权变化或签名撤回会丢弃这次答复和 编译后的代码。编译过程无法中断，但已过期或已取消的请求不会继续运行。 @@ -86,22 +126,27 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 | 表元素 | 16,384 个 | | wasm 栈 | 512 KiB | | 模块 | 8 MiB | -| 一次调用的输入或输出 | 运行时 16 MiB；服务中序列化后的输入 1 MiB | +| 一次调用的输入或输出 | 运行时 16 MiB；服务中序列化后的输入 8 MiB | | 日志 | 每次调用 64 行，每行 1 KiB | | 每个应用的模块 | 8 个 | | 同时运行函数的应用 | 4 个工作线程，每个应用一个 | | 排队的请求 | 每个应用 4 个 | -| 所有应用缓冲的输入 | 16 MiB | +| 所有应用缓冲的输入 | 32 MiB | | 一个请求（包括排队和加载） | 10 秒 | -| 空闲的工作线程 | 5 秒后退出 | +| 空闲的工作线程 | 5 秒后退出；持有组件实例时 60 秒后退出 | 队列已满或没有空闲的工作线程时，请求会立即失败，不会等待。 +App Hub 安装或更新应用时，外壳会在后台把它的函数编译进缓存（`wasm_service::warm`）：一次一个 +应用，在单独的线程上，只编译已准入的应用包，并且只在应用有 `wasm` 授权时进行。这样应用的第一次 +调用就从缓存加载，不必等待 Cranelift 编译。 + ### Agent 工具 应用的 `tools.json` 可以用 `"host_method": "wasm."` 把一个工具映射到自己的 某个函数。这需要 `wasm` 能力。与共享宿主方法不同，它没有最低风险等级，也不需要 -`"private_data": true`，因为函数只能看到传给它的参数。Wasm Lab 把 `wasmlab.find_slots`、 +`"private_data": true`：模块只能看到传给它的参数，组件还能看到自己应用的文件夹，与应用的 +脚本相同。两者都接触不到共享数据。Wasm Lab 把 `wasmlab.find_slots`、 `wasmlab.rank` 和 `wasmlab.diff` 映射到 `wasm.find_slots`、`wasm.fuzzy_rank` 和 `wasm.text_diff`，风险等级都是 `read`。 @@ -112,9 +157,9 @@ OctoSense 中有四处用到 WebAssembly。其中一处专为运行应用自己 | macOS | 运行。已在隐藏的桌面外壳中验证（ADR 0011）。Wasmtime 在这里用 Mach 异常端口而不是信号捕获陷阱。 | | Linux | 运行。运行时和服务的测试在 x86_64 上通过，陷阱由信号捕获；Wasm Lab 在无头桌面中运行过（2026 年 10 月 8 日）。 | | Android | 在 Home 的默认构建中运行。需要 libc 0.2.190 或更高版本，`crates/wasm-host` 已要求这一点：在 0.2.189 下，每个陷阱都会结束进程。已在 Redmi Note 12 上验证（ADR 0011），并在一部 OnePlus 6 上通过[手机验收](#手机验收)。运行时增加约 7.5 MiB 代码；在 Snapdragon 685 上编译一个 433 KiB 的模块约需 0.4 秒，从缓存加载需 5–11 毫秒。 | -| Windows | 不包含。在 Windows 上检查过之前，运行时不进入构建。 | -| iOS | 不包含。那里的应用不允许 JIT；可选方案是 Wasmtime 的解释器 Pulley，比 Cranelift 慢约 17 倍。 | -| OpenHarmony | 不包含。其 JIT 策略**未验证**。 | +| Windows | 包含（ADR 0014）。[`wasm-windows.yml`](../.github/workflows/wasm-windows.yml) 在 GitHub 的 `windows-2022` 上运行运行时的测试，并编译带这个服务的外壳。在 Windows 上运行桌面版**未验证**。 | +| iOS | 不包含。那里的应用不能生成代码，所以运行时会像在 OpenHarmony 上一样使用 Pulley；但还没有编译过带这个服务的 iOS 构建。 | +| OpenHarmony | 包含，在 OpenHarmony 的代码生成策略明确之前于 Wasmtime 的解释器 Pulley 中运行：Cranelift 编译为 Pulley 字节码，没有任何东西以原生代码运行。Home 的 OpenHarmony 发布构建已带着这个服务编译通过（2026 年 10 月 9 日，`cargo-makepad makepad ohos … deveco -p octosense-home --release`）；还没有在设备上运行过（**未验证**）。Pulley 比 Cranelift 慢约 32 倍（ADR 0014）。 | 在受管理的 Mac 上，Microsoft Defender 会把新写入的缓存文件的第一次打开拦住约一秒；具体 测量见 ADR 0011。 @@ -160,8 +205,24 @@ cd phone && cargo test --locked --features mobile-apps -p octosense-shell wasm_s 并发写入、取消。 - `crates/wasm-host/tests/guest.rs` 运行 Wasm Lab 的模块，把每个函数的结果与同一段 Rust 原生运行的结果比较，然后逐一运行 `rogue` 的各种模式。 +- `crates/wasm-host/tests/component.rs` 运行四个由 `crates/wasm-host/tests/component-guest` + 构建的组件：以 JSON 调用未作修改的 crate（pulldown-cmark）、记录与 snake_case 名称、 + 同一实例保留状态、时钟、随机数与字节、只能访问授予的文件夹（也包括只读）、存储预算、 + 超时与内存上限；`fetch`，它的 `wasi:http` 只有在主机获准时才能访问本地服务器，向从不应答的 + 服务器发出的请求在截止时间结束；`hostcall`，它通过嵌入方调用宿主服务；以及拒绝导入套接字的组件。 - 服务测试覆盖工具映射、输入上限、每次调用使用全新实例、更新或授权变化或撤回之后的撤销、 - 取消和队列上限，以及应用自己的函数回答它的工具和脚本。 + 取消和队列上限，以及应用自己的函数回答它的工具和脚本。对组件，测试覆盖带类型的调用、 + 实例的状态、应用的存储文件夹及其配额、没有存储能力的应用、作为系统应用经 App Hub 准入发布的 + 组件、只能访问应用主机的请求、只能到达已授权服务的宿主调用，以及已安装应用的第一次调用从缓存加载。 +- `cargo run --release -p octosense-wasm-host --example measure_component` 打印组件 + 相对于模块和原生 Rust 的耗时。 +- `OCTOSENSE_WASM_PULLEY=1 cargo test --locked -p octosense-wasm-host --features pulley` + 在 Pulley 中运行同样的测试，与 OpenHarmony 的运行方式相同；`phone.yml` 也运行它。 + +测试组件可以逐字节重建：用 rustc 1.97.1，`crates/wasm-host/tests/component-guest/build.sh` +写出相同的 `notes.component.wasm`（SHA-256 `963c965b…`）、`netprobe.component.wasm` +（`98b2402a…`）、`fetch.component.wasm`（`6eaf6ac8…`）和 `hostcall.component.wasm` +（`d39f2a8e…`）。只需添加一次目标：`rustup target add wasm32-wasip2`。 `phone.yml` 运行第一条命令，服务的测试则作为 Home 测试的一部分运行（`mobile-apps` 包含 `wasm-functions`）；`tools/ci-local.sh --linux-host --offload` 在 Linux 构建主机上运行 @@ -184,11 +245,11 @@ cd phone && cargo test --locked --features mobile-apps -p octosense-shell wasm_s - 还没有任何发布版本包含这个服务：#400 之后从 `main` 构建的第一个桌面版和 Home 发布版 将会包含。 -- Windows、iOS（Pulley）和 OpenHarmony。 +- iOS：带这个服务（在 Pulley 中）构建 Home。 - 应用之间的 CPU 公平调度，以及磁盘缓存的上限：目前没有任何东西会清理它。 -- 在安装时（商店应用）或构建时（系统应用）编译，省掉手机上的第一次编译。 -- 用类型化接口（组件模型和 WIT）代替 JSON。`octo.log` 以外的任何导入，例如时钟或随机数， - 都将是新的能力。 +- 提前编译系统应用的函数：已安装应用的函数在安装时编译，系统应用的函数在第一次调用时编译。 +- 手机上的组件：Android 和 OpenHarmony 构建链接了运行时，但还没有组件在设备上运行过（**未验证**）。 +- 经 JSON 传字节很慢：1 MiB 的 `list` 以 base64 往返约需 18 毫秒。 ## 引擎插件：`photo` 和 `vector` 服务 diff --git a/native-apps.json b/native-apps.json index b48d82d11..201bb79b2 100644 --- a/native-apps.json +++ b/native-apps.json @@ -114,7 +114,7 @@ "id": "apphub", "feature": "app-hub", "crate": "octosense-app-hub-app", - "source": { "git": "https://github.com/OctoSense-org/OctoSense-App-Hub", "rev": "28745c0f6fd71986820e9835183a3045f61baefc", "local": null }, + "source": { "git": "https://github.com/OctoSense-org/OctoSense-App-Hub", "rev": "371f51aeb6ba90e9a214c5ddb6ccfc40a2e2a339", "local": null }, "module": ["octosense_app_hub_app::APP_HUB_MODULE", "octosense_app_hub_app::CARD_MODULE"], "bin": null, "default_features": true, diff --git a/tools/check-shell-graph.sh b/tools/check-shell-graph.sh index 48fc4d9bb..cd2f1ed34 100755 --- a/tools/check-shell-graph.sh +++ b/tools/check-shell-graph.sh @@ -55,7 +55,7 @@ fail() { echo "::error::$*"; exit 1; } # The targets the `wasm` service runs on (crates/shell/Cargo.toml, build.rs). wasm_runs_on() { case $1 in - *-apple-darwin | *-linux-android* | *-unknown-linux-gnu* | *-unknown-linux-musl*) return 0 ;; + *-apple-darwin | *-pc-windows-* | *-linux-android* | *-unknown-linux-gnu* | *-unknown-linux-musl* | *-unknown-linux-ohos*) return 0 ;; *) return 1 ;; esac } @@ -149,12 +149,12 @@ for features in "${feature_sets[@]}"; do done # Where the wasm service does not run, its runtime is not even built. -for target in aarch64-apple-ios aarch64-unknown-linux-ohos x86_64-pc-windows-msvc; do +for target in aarch64-apple-ios; do if linked octosense-wasm-host --locked ${manifest[@]+"${manifest[@]}"} ${package[@]+"${package[@]}"} --target "$target"; then fail "octosense-wasm-host is linked for $target, where the wasm service does not run" fi done -echo "ok: no wasm runtime for iOS, OpenHarmony or Windows" +echo "ok: no wasm runtime for iOS" # One octos: every octos-* crate in the lock from one octos-org/octos rev # (the `nix` patch taken from the octos repo is not an octos crate). diff --git a/tools/ci_local.py b/tools/ci_local.py index 2ca6c6d53..e60f61a00 100644 --- a/tools/ci_local.py +++ b/tools/ci_local.py @@ -82,6 +82,8 @@ "compilation or another OS is not native account evidence", "embedded-browser.yml": "native Windows WebView2 acceptance needs a Windows desktop and installed engine; " "ci-local-merge requires a successful GitHub run on the exact PR head", + "wasm-windows.yml": "apps' WebAssembly functions on Windows need a Windows runner; " + "ci-local-merge requires a successful GitHub run on the exact PR head", "release-desktop.yml": "a release workflow (tag push or manual run): signed packages for three OSes, " "with a matrix, environments, secrets and artifacts; nothing a pull request merges " "on depends on it, and its packaging scripts' tests run in desktop.yml",