diff --git a/build/deps/rust.MODULE.bazel b/build/deps/rust.MODULE.bazel index 54128840bac..4f481b119c9 100644 --- a/build/deps/rust.MODULE.bazel +++ b/build/deps/rust.MODULE.bazel @@ -97,6 +97,14 @@ crate.annotation( "//:patches/rust/crates/hyper/parse-error-explanation.patch", ], ) +crate.annotation( + # rust_icu's checked-in bindings and symbol-renaming macro are selected by this value. Keep it + # synchronized with U_ICU_VERSION_MAJOR_NUM of the Chromium ICU commit pinned in + # build/deps/gen/deps.MODULE.bazel. ci/v8_update.py updates it; see docs/v8-updates.md. + build_script_env = {"BAZEL": "1"}, + crate = "rust_icu_sys", + rustc_env = {"RUST_ICU_MAJOR_VERSION_NUMBER": "78"}, +) crate.from_cargo( # TODO(cleanup): This name is a bit misleading now that we don't vendor. We can clean it up as a # followup. diff --git a/build/wd_rust_crate.bzl b/build/wd_rust_crate.bzl index 81ab075ff65..266f09659aa 100644 --- a/build/wd_rust_crate.bzl +++ b/build/wd_rust_crate.bzl @@ -66,6 +66,7 @@ def wd_rust_crate( deps = [], link_deps = [], proc_macro_deps = [], + rustc_env = {}, data = [], test_env = {}, test_tags = [], @@ -174,6 +175,7 @@ def wd_rust_crate( srcs = srcs, deps = deps, link_deps = link_deps + ["@@//deps:rust_runtime"], + rustc_env = rustc_env, visibility = visibility, data = data, proc_macro_deps = proc_macro_deps, @@ -195,6 +197,7 @@ def wd_rust_crate( size = test_size, tags = test_tags, crate_features = crate_features, + rustc_env = rustc_env, deps = test_deps, proc_macro_deps = test_proc_macro_deps, ) diff --git a/ci/v8_update.py b/ci/v8_update.py index f0e6758439f..23929f9a9e0 100644 --- a/ci/v8_update.py +++ b/ci/v8_update.py @@ -35,6 +35,14 @@ # Keep the aligned markers in sync with the corresponding comments in # build/deps/deps.jsonc. +# rust_icu_sys selects its bindings and versioned ICU symbol names from this +# pin, which must match the major version of the ICU that V8 links. +RUST_ICU_VERSION_PINS = ( + ROOT / "build/deps/rust.MODULE.bazel", + ROOT / "src/rust/i18n/BUILD.bazel", +) +RUST_ICU_VERSION = re.compile(r'(?<="RUST_ICU_MAJOR_VERSION_NUMBER": ")\d+(?=")') + def run(args, *, cwd=ROOT, check=True, env=None, stdout=None): print("+", shlex.join(map(str, args)), flush=True) @@ -103,6 +111,41 @@ def _update_aligned_dependencies(): DEPS.write_text(jsonc.dumps(doc) + "\n") +def _icu_major_version(commit): + with urlopen( + "https://chromium.googlesource.com/chromium/deps/icu/+/" + f"{commit}/source/common/unicode/uvernum.h?format=TEXT", + timeout=30, + ) as response: + header = base64.b64decode(response.read()).decode() + match = re.search( + r"(?<=^#define U_ICU_VERSION_MAJOR_NUM )\d+$", header, re.MULTILINE + ) + return match.group() + + +def _update_rust_icu_version(): + path, _ = V8_DEPENDENCIES["com_googlesource_chromium_icu"] + commit = _v8_dependency_commit(path, (CHECKOUT / "DEPS").read_text()) + version = _icu_major_version(commit) + changed = False + for pin in RUST_ICU_VERSION_PINS: + text = pin.read_text() + old = RUST_ICU_VERSION.findall(text) + if len(old) != 1: + raise ValueError( + f"expected one RUST_ICU_MAJOR_VERSION_NUMBER in {pin}, found {len(old)}" + ) + if old != [version]: + pin.write_text(RUST_ICU_VERSION.sub(version, text)) + changed = True + if changed: + print( + f"Updated RUST_ICU_MAJOR_VERSION_NUMBER to {version}. Check that the pinned " + f"rust_icu_sys provides bindgen/lib_{version}.rs; update the crate if it does not." + ) + + def _update_module(tag, patch_names): text = MODULE.read_text() text = re.sub( @@ -220,6 +263,16 @@ def finish_update(target): _update_module(target, patch_names) _update_aligned_dependencies() + # The Rust ICU pin must match the ICU that Bazel builds, so regenerate the + # ICU repository from its new commit before deriving the pin. + run( + [ + sys.executable, + ROOT / "build/deps/update-deps.py", + "com_googlesource_chromium_icu", + ] + ) + _update_rust_icu_version() def parse_args(): diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index 2b6a6f641ae..d63e35198b4 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -595,6 +595,7 @@ dependencies = [ "rand 0.9.5", "ruff_python_ast", "ruff_python_parser", + "rust_icu_sys", "rustls", "rustls-platform-verifier", "rustls-webpki", @@ -1321,6 +1322,12 @@ dependencies = [ "libc", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.190" @@ -1534,6 +1541,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1881,6 +1894,28 @@ name = "ruff_text_size" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff?tag=0.12.1#32c54189cb45a9d0409a1140265ce6d5fcec214d" +[[package]] +name = "rust_icu_release" +version = "5.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adff2447072626b05450f000578265104ba9df89e67e781ee54add74abed97e8" +dependencies = [ + "anyhow", +] + +[[package]] +name = "rust_icu_sys" +version = "5.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bf2afe1327bd09e8d50bcf8abcea97410a10b2f10b932e994abad414038a711" +dependencies = [ + "anyhow", + "lazy_static", + "libc", + "paste", + "rust_icu_release", +] + [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index 52d582ca696..e802da86d40 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -76,6 +76,9 @@ pico-args = "0" rand = { version = "0.9", default-features = false, features = ["os_rng", "thread_rng"] } ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } +# Use rust_icu's checked-in bindings for the Chromium ICU version linked by workerd. ICU itself is +# supplied by Bazel, so bindgen, icu-config, and rust_icu's library-linking features stay disabled. +rust_icu_sys = { version = "5.8", default-features = false, features = ["renaming", "icu_version_in_env"] } # ring rather than aws-lc-rs: its crates.io package builds with plain cc (no cmake/NASM). rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-platform-verifier = "0.7" diff --git a/docs/v8-updates.md b/docs/v8-updates.md index f7fd3848b67..89451b53a2f 100644 --- a/docs/v8-updates.md +++ b/docs/v8-updates.md @@ -84,6 +84,13 @@ from the V8 directory. figure out what version of the github code V8 depends on. Instead, it should be safe to just bump these dependencies to the latest version on github. + If the new `com_googlesource_chromium_icu` commit changes `U_ICU_VERSION_MAJOR_NUM` + (in `source/common/unicode/uvernum.h`), update `RUST_ICU_MAJOR_VERSION_NUMBER` in both + `build/deps/rust.MODULE.bazel` and `src/rust/i18n/BUILD.bazel`. A stale value fails the + build with undefined `_` ICU symbols, or missing items in + `rust_icu_sys` if the two values disagree. If `rust_icu_sys` has no `bindgen/lib_.rs` + for the new version, update the crate as well. + 11. Check workerd's tests pass with the updated V8. ```sh @@ -122,6 +129,10 @@ rebases the patches, regenerates `patches/v8/*.patch`, and updates: - the `dragonbox`, `fast_float`, `fp16`, and `highway` commits in `build/deps/deps.jsonc`. +It also regenerates `com_googlesource_chromium_icu` in `build/deps/gen/deps.MODULE.bazel` +and sets `RUST_ICU_MAJOR_VERSION_NUMBER` in `build/deps/rust.MODULE.bazel` and +`src/rust/i18n/BUILD.bazel` to the major version of the new ICU commit (see step 10). + If the rebase stops because of conflicts, resolve and continue it in the temporary checkout: @@ -143,7 +154,7 @@ python3 ci/v8_update.py finish Do not run `finish` until the rebase has completed. It regenerates the patches and updates the pins listed above. -The helper does not run `update-deps.py`. Compare the old and new V8 `DEPS` files for +The helper runs `update-deps.py` only for ICU. Compare the old and new V8 `DEPS` files for changes to `dragonbox`, `fast_float`, `fp16`, `highway`, `perfetto`, and `simdutf`, then run the dependency updater for each changed revision: diff --git a/src/rust/i18n/BUILD.bazel b/src/rust/i18n/BUILD.bazel new file mode 100644 index 00000000000..f935fc9b1f0 --- /dev/null +++ b/src/rust/i18n/BUILD.bazel @@ -0,0 +1,24 @@ +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +wd_rust_crate( + name = "i18n", + cxx_bridge_deps = [ + "//src/rust/jsg", + "@simdutf", + "@workerd-v8//:v8", + ], + cxx_bridge_src = "ffi.rs", + # ICU, which rust_icu_sys binds to, is linked through V8. + link_deps = ["@workerd-v8//:v8"], + # rust_icu_sys's `versioned_function!` expands in this crate and reads the ICU version from + # here. Keep it synchronized with the rust_icu_sys annotation in build/deps/rust.MODULE.bazel; + # ci/v8_update.py updates both. + rustc_env = {"RUST_ICU_MAJOR_VERSION_NUMBER": "78"}, + test_deps = ["//src/rust/jsg-test"], + visibility = ["//visibility:public"], + deps = [ + "//src/rust/jsg", + "@crates_vendor//:rust_icu_sys", + "@crates_vendor//:thiserror", + ], +) diff --git a/src/rust/i18n/dispatch-test.rs b/src/rust/i18n/dispatch-test.rs new file mode 100644 index 00000000000..e4e4b3ed2c8 --- /dev/null +++ b/src/rust/i18n/dispatch-test.rs @@ -0,0 +1,180 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use jsg_test::Harness; + +use super::*; + +/// All four transcodable encodings, for exhaustively testing every pair. +const ENCODINGS: [Encoding; 4] = [ + Encoding::Ascii, + Encoding::Latin1, + Encoding::Utf8, + Encoding::Utf16Le, +]; + +// `Harness::new()` initializes the V8 platform, which is what installs the +// embedded ICU data. ICU converter opens fail without it, even though these +// tests never create an isolate or run JavaScript. +fn init_icu() -> Harness { + Harness::new() +} + +/// Runs both transcode steps the way [`crate::transcode`] does -- size, +/// allocate, convert, narrow to the written length -- against a plain +/// `Vec` rather than a V8 backing store, and returns the written bytes. +fn transcode(source: &[u8], from: Encoding, to: Encoding) -> Result, TranscodeError> { + let transcoder = Transcoder::new(source, from, to)?; + let mut dest = vec![0u8; transcoder.dest_len()]; + let written = transcoder.transcode_into(&mut dest)?; + assert!( + written <= dest.len(), + "{from:?} -> {to:?} wrote {written} bytes into a {} byte buffer", + dest.len() + ); + dest.truncate(written); + Ok(dest) +} + +#[test] +fn every_pair_round_trips_ascii_text() { + let _harness = init_icu(); + for &from in &ENCODINGS { + // UTF16LE source bytes must have even length; every other encoding + // is happy with plain ASCII bytes. + let source: &[u8] = if from == Encoding::Utf16Le { + &[0x48, 0x00, 0x69, 0x00] // "Hi" as UTF-16LE code units. + } else { + b"Hi" + }; + for &to in &ENCODINGS { + let result = transcode(source, from, to); + assert!(result.is_ok(), "{from:?} -> {to:?} failed: {result:?}"); + } + } +} + +#[test] +fn every_pair_empty_input_is_empty_output() { + let _harness = init_icu(); + for &from in &ENCODINGS { + for &to in &ENCODINGS { + let result = transcode(&[], from, to); + assert_eq!( + result.as_deref(), + Ok([].as_slice()), + "{from:?} -> {to:?} on empty input should be empty, got {result:?}" + ); + } + } +} + +#[test] +fn identity_pairs_round_trip() { + let _harness = init_icu(); + for &encoding in &ENCODINGS { + // UTF16LE source bytes must be well-formed UTF-16LE to pass through + // unchanged. + let source: &[u8] = if encoding == Encoding::Utf16Le { + b"H\0i\0" + } else { + b"identity" + }; + assert_eq!( + transcode(source, encoding, encoding).unwrap(), + source, + "{encoding:?} identity" + ); + } +} + +#[test] +fn unmappable_characters_become_question_marks() { + let _harness = init_icu(); + // 'ā˜•' (U+2615, HOT BEVERAGE) has no representation in ASCII or Latin-1. + let utf8_source = "ā˜•".as_bytes(); + let utf16le_source = [0x15, 0x26]; + for (source, from) in [ + (utf8_source, Encoding::Utf8), + (utf16le_source.as_slice(), Encoding::Utf16Le), + ] { + for to in [Encoding::Ascii, Encoding::Latin1] { + assert_eq!( + transcode(source, from, to).unwrap(), + b"?", + "{from:?} -> {to:?}" + ); + } + } +} + +#[test] +fn ascii_to_utf16le_widens_high_bytes_via_latin1_path() { + // ASCII -> UTF16LE takes the Latin-1 path, so a high byte is widened to + // U+00FF rather than substituted with '?'. + let result = transcode(&[0xff], Encoding::Ascii, Encoding::Utf16Le).unwrap(); + assert_eq!(result, vec![0xff, 0x00]); +} + +#[test] +fn utf8_continuation_byte_only_input_yields_empty_utf16le() { + // The UTF8 -> UTF16LE size estimate is zero for input consisting only of + // continuation bytes, so the result is empty rather than an error, even + // though the input is non-empty. + let result = transcode(&[0x80], Encoding::Utf8, Encoding::Utf16Le).unwrap(); + assert!(result.is_empty()); +} + +#[test] +fn invalid_utf8_to_utf16le_is_unable_to_transcode() { + let result = transcode(&[0x61, 0xc3], Encoding::Utf8, Encoding::Utf16Le); + assert_eq!(result, Err(TranscodeError::UnableToTranscode)); +} + +#[test] +fn odd_length_utf16le_to_utf8_is_rejected() { + let result = transcode(&[0x61], Encoding::Utf16Le, Encoding::Utf8); + assert_eq!(result, Err(TranscodeError::OddUtf16leInput)); +} + +#[test] +fn odd_length_utf16le_to_latin1_is_rejected() { + let _harness = init_icu(); + let result = transcode(&[0x61], Encoding::Utf16Le, Encoding::Latin1); + assert_eq!(result, Err(TranscodeError::OddUtf16leInput)); +} + +#[test] +fn unpaired_surrogate_utf16le_to_utf8() { + // U+D800, an unpaired high surrogate, encoded as UTF-16LE bytes. + let source = [0x00, 0xd8]; + // `utf8_length_from_utf16le` does not validate, and reports a nonzero + // length for the surrogate, but `convert_utf16le_to_utf8` rejects the input + // and writes nothing -- so the mismatch check is what surfaces the + // failure. + let result = transcode(&source, Encoding::Utf16Le, Encoding::Utf8); + assert_eq!(result, Err(TranscodeError::Utf8LengthMismatch)); +} + +#[test] +fn out_of_range_source_encoding_is_rejected() { + let invalid = Encoding { repr: 4 }; + for &to in &ENCODINGS { + assert_eq!( + Transcoder::new(b"Hi", invalid, to).err(), + Some(TranscodeError::InvalidEncoding), + "{to:?}" + ); + } +} + +#[test] +fn destination_size_mismatch_is_rejected() { + let transcoder = Transcoder::new(b"Hi", Encoding::Latin1, Encoding::Utf16Le).unwrap(); + let mut too_small = vec![0u8; transcoder.dest_len() - 1]; + assert_eq!( + transcoder.transcode_into(&mut too_small), + Err(TranscodeError::DestinationSizeMismatch) + ); +} diff --git a/src/rust/i18n/dispatch.rs b/src/rust/i18n/dispatch.rs new file mode 100644 index 00000000000..6666bd552ff --- /dev/null +++ b/src/rust/i18n/dispatch.rs @@ -0,0 +1,293 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Encoding-pair dispatch and the conversion logic behind each pair, ported +//! from `i18n.c++`'s `TranscodeDefault` / `TranscodeLatin1ToUTF16` / +//! `TranscodeFromUTF16` / `TranscodeUTF16FromUTF8` / `TranscodeUTF8FromUTF16` +//! and the `switch` in `transcode()` that picks between them. +//! +//! A transcode runs in two steps. [`Transcoder::new`] validates the source, +//! picks the conversion, and computes [`Transcoder::dest_len`] -- the exact +//! size of the destination buffer that conversion needs. +//! [`Transcoder::transcode_into`] then fills a buffer of that size and reports +//! how many bytes it actually wrote, which is often fewer because the +//! destination is sized for the worst case. +//! +//! All sizing, validation, substitute-character setup, and length checking +//! happens here; the ICU and simdutf wrappers in [`crate::ffi`] only provide +//! the conversions themselves. + +use crate::error::TranscodeError; +use crate::ffi; +use crate::ffi::Converter; +use crate::ffi::Encoding; +use crate::ffi::Utf8Source; +use crate::ffi::Utf16LeSource; + +/// An isolate has a 128MB memory limit, and thus so does any single +/// destination buffer. Mirrors `ISOLATE_LIMIT` in `i18n.c++`. +const ISOLATE_LIMIT: usize = 134_217_728; + +/// Fails with `error` unless `limit` is within [`ISOLATE_LIMIT`], mirroring +/// `i18n.c++`'s `JSG_REQUIRE(limit <= ISOLATE_LIMIT, Error, ...)`. +/// +/// Callers compute `limit` with saturating arithmetic, where the C++ path +/// multiplies unchecked, so an overflowing size saturates and fails here. +const fn require_within_isolate_limit( + limit: usize, + error: TranscodeError, +) -> Result<(), TranscodeError> { + if limit <= ISOLATE_LIMIT { + Ok(()) + } else { + Err(error) + } +} + +/// A validated, sized transcode, ready to run. +/// +/// Borrows its source for the whole of its life, so [`Transcoder::dest_len`] +/// cannot go stale: the bytes it was computed from are the same bytes +/// [`Transcoder::transcode_into`] reads. The UTF-8/UTF-16 conversions check +/// that they wrote exactly the length estimated from the source, so a stale +/// estimate would turn into a spurious length-mismatch error. +pub struct Transcoder<'a> { + source: &'a [u8], + conversion: Conversion<'a>, + dest_len: usize, +} + +/// The conversion [`Transcoder::transcode_into`] will perform, along with any +/// ICU converters or measured simdutf sources [`Transcoder::new`] needed to +/// size the destination. +enum Conversion<'a> { + /// ICU `ucnv_convertEx` between two converters, mirroring + /// `TranscodeDefault`. Handles every pair the conversions below do not, + /// including all four identity pairs. + ConvertEx { to: Converter, from: Converter }, + /// Latin-1 to UTF-16, mirroring `TranscodeLatin1ToUTF16`. + Latin1ToUtf16, + /// ICU `ucnv_fromUChars` from UTF-16LE into ASCII or Latin-1, mirroring + /// `TranscodeFromUTF16`. + FromUtf16 { to: Converter }, + /// UTF-8 to UTF-16LE, mirroring `TranscodeUTF16FromUTF8`. + Utf16FromUtf8(Utf8Source<'a>), + /// UTF-16LE to UTF-8, mirroring `TranscodeUTF8FromUTF16`. + Utf8FromUtf16(Utf16LeSource<'a>), +} + +impl<'a> Transcoder<'a> { + /// Prepares a transcode of `source` from `from` to `to`, matching the + /// dispatch table built by `i18n::transcode()` in `i18n.c++`. + /// + /// Returns an error if `from` is not a transcodable encoding, if `source` + /// is malformed for `from`, or if the destination the conversion would + /// need exceeds [`ISOLATE_LIMIT`]. + pub fn new(source: &'a [u8], from: Encoding, to: Encoding) -> Result { + match from { + Encoding::Ascii | Encoding::Latin1 => { + if to == Encoding::Utf16Le { + Self::transcode_latin1_to_utf16(source) + } else { + Self::transcode_default(source, from, to) + } + } + Encoding::Utf8 => { + if to == Encoding::Utf16Le { + Self::transcode_utf16_from_utf8(source) + } else { + Self::transcode_default(source, from, to) + } + } + Encoding::Utf16Le => match to { + Encoding::Utf16Le => Self::transcode_default(source, from, to), + Encoding::Utf8 => Self::transcode_utf8_from_utf16(source), + _ => Self::transcode_from_utf16(source, to), + }, + _ => Err(TranscodeError::InvalidEncoding), + } + } + + /// The exact size, in bytes, of the destination buffer + /// [`Transcoder::transcode_into`] requires. + pub fn dest_len(&self) -> usize { + self.dest_len + } + + /// Transcodes into `dest`, returning the number of bytes written, which + /// may be fewer than `dest.len()`. + /// + /// `dest` must be exactly [`Transcoder::dest_len`] bytes long. + pub fn transcode_into(&self, dest: &mut [u8]) -> Result { + // The conversions below refuse a short destination or stop at its end + // rather than overflowing it, but the result would be wrong. + if dest.len() != self.dest_len { + return Err(TranscodeError::DestinationSizeMismatch); + } + // A zero-length destination means the conversion has nothing to write: + // either the source was empty, or the estimated output length was zero. + if dest.is_empty() { + return Ok(0); + } + + let source = self.source; + match &self.conversion { + Conversion::ConvertEx { to, from } => { + ffi::convert_ex(to, from, source, dest).ok_or(TranscodeError::UnableToTranscode) + } + Conversion::Latin1ToUtf16 => { + // Every byte is valid Latin-1 and widens to exactly one UTF-16 + // code unit, and `dest` was sized as two bytes per source byte. + // The C++ path checks simdutf's result for 0, which it can only + // return for an empty source, handled above. + let units = ffi::convert_latin1_to_utf16(source, dest) + .ok_or(TranscodeError::DestinationSizeMismatch)?; + Ok(units * size_of::()) + } + Conversion::FromUtf16 { to } => { + ffi::from_uchars(to, source, dest).ok_or(TranscodeError::UnableToTranscode) + } + Conversion::Utf16FromUtf8(utf8) => { + // `dest` was sized as two bytes per estimated code unit. + let expected_units = utf8.utf16_len(); + let units = utf8 + .convert_to_utf16le(dest) + .ok_or(TranscodeError::DestinationSizeMismatch)?; + // 0 means invalid UTF-8 input. + if units == 0 { + return Err(TranscodeError::UnableToTranscode); + } + if units != expected_units { + return Err(TranscodeError::Utf16LengthMismatch); + } + Ok(dest.len()) + } + Conversion::Utf8FromUtf16(utf16) => { + let expected_bytes = utf16.utf8_len(); + let written = utf16 + .convert_to_utf8(dest) + .ok_or(TranscodeError::DestinationSizeMismatch)?; + // Invalid input writes 0 bytes, which fails this check because + // `expected_bytes` is nonzero here. The C++ + // `TranscodeUTF8FromUTF16` checks for 0 only *after* requiring + // equality with the (nonzero) estimate, so that branch is dead + // there too: invalid input surfaces as a length mismatch, not + // "Unable to transcode buffer", unlike every other pair. + if written != expected_bytes { + return Err(TranscodeError::Utf8LengthMismatch); + } + Ok(written) + } + } + } + + /// Mirrors `TranscodeDefault`: ICU `ucnv_convertEx` between two + /// converters, substituting `?` for unmappable characters, sized at `to`'s + /// maximum bytes per character for every source byte. + fn transcode_default( + source: &'a [u8], + from: Encoding, + to: Encoding, + ) -> Result { + let to_conv = Converter::open(to)?; + to_conv.set_subst_chars(&"?".repeat(to_conv.min_char_size()))?; + let from_conv = Converter::open(from)?; + + let limit = source.len().saturating_mul(to_conv.max_char_size()); + require_within_isolate_limit(limit, TranscodeError::SourceBufferTooLarge)?; + + Ok(Self { + source, + conversion: Conversion::ConvertEx { + to: to_conv, + from: from_conv, + }, + dest_len: limit, + }) + } + + /// Mirrors `TranscodeLatin1ToUTF16`: widening of ASCII/Latin-1 into + /// UTF-16. + /// + /// Taken for an `Ascii` source as well as a `Latin1` one, so source bytes + /// `0x80`-`0xFF` widen to U+0080-U+00FF instead of being substituted. + fn transcode_latin1_to_utf16(source: &'a [u8]) -> Result { + let length_in_chars = source.len().saturating_mul(2); + require_within_isolate_limit(length_in_chars, TranscodeError::SourceBufferTooLarge)?; + + Ok(Self { + source, + conversion: Conversion::Latin1ToUtf16, + dest_len: length_in_chars, + }) + } + + /// Mirrors `TranscodeFromUTF16`: ICU `ucnv_fromUChars` from UTF-16LE into + /// ASCII or Latin-1, substituting `?` for unmappable characters, sized at + /// `to`'s maximum bytes per character for every code unit. + fn transcode_from_utf16(source: &'a [u8], to: Encoding) -> Result { + let to_conv = Converter::open(to)?; + to_conv.set_subst_chars(&"?".repeat(to_conv.min_char_size()))?; + + if !source.len().is_multiple_of(2) { + return Err(TranscodeError::OddUtf16leInput); + } + + let limit = (source.len() / 2).saturating_mul(to_conv.max_char_size()); + require_within_isolate_limit(limit, TranscodeError::BufferTooLarge)?; + + Ok(Self { + source, + conversion: Conversion::FromUtf16 { to: to_conv }, + dest_len: limit, + }) + } + + /// Mirrors `TranscodeUTF16FromUTF8`: UTF-8 to UTF-16LE, sized from + /// [`Utf8Source::utf16_len`]. + /// + /// That estimate is zero for some non-empty inputs -- a source of nothing + /// but UTF-8 continuation bytes, for instance -- which yields an empty + /// result rather than an error. + fn transcode_utf16_from_utf8(source: &'a [u8]) -> Result { + let utf8 = Utf8Source::new(source); + let expected_utf16_length = utf8.utf16_len(); + require_within_isolate_limit( + expected_utf16_length, + TranscodeError::ExpectedUtf16LengthTooLarge, + )?; + + Ok(Self { + source, + conversion: Conversion::Utf16FromUtf8(utf8), + // Cannot overflow: at most `ISOLATE_LIMIT` code units. + dest_len: expected_utf16_length * size_of::(), + }) + } + + /// Mirrors `TranscodeUTF8FromUTF16`: UTF-16LE to UTF-8, sized from + /// [`Utf16LeSource::utf8_len`]. + fn transcode_utf8_from_utf16(source: &'a [u8]) -> Result { + if !source.len().is_multiple_of(2) { + return Err(TranscodeError::OddUtf16leInput); + } + + let utf16 = Utf16LeSource::new(source); + let expected_utf8_length = utf16.utf8_len(); + require_within_isolate_limit( + expected_utf8_length, + TranscodeError::ExpectedUtf8LengthTooLarge, + )?; + + Ok(Self { + source, + conversion: Conversion::Utf8FromUtf16(utf16), + dest_len: expected_utf8_length, + }) + } +} + +#[cfg(test)] +#[path = "dispatch-test.rs"] +mod tests; diff --git a/src/rust/i18n/error-test.rs b/src/rust/i18n/error-test.rs new file mode 100644 index 00000000000..81d014757ea --- /dev/null +++ b/src/rust/i18n/error-test.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use jsg::ExceptionType; + +use super::*; + +#[test] +fn allocation_failure_is_a_range_error() { + let error = jsg::Error::from(TranscodeError::AllocationFailed); + assert_eq!(error.name, ExceptionType::RangeError); + assert_eq!(error.message, "Failed to allocate memory for Uint8Array"); +} + +#[test] +fn other_failures_are_plain_errors() { + let error = jsg::Error::from(TranscodeError::OddUtf16leInput); + assert_eq!(error.name, ExceptionType::Error); +} diff --git a/src/rust/i18n/error.rs b/src/rust/i18n/error.rs new file mode 100644 index 00000000000..bac1a8ec5df --- /dev/null +++ b/src/rust/i18n/error.rs @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The error type reported by [`crate::dispatch`], and its conversion into the +//! JavaScript `Error` the caller of `node:buffer`'s `transcode()` sees. + +use thiserror::Error; + +/// A failed transcode. +/// +/// The messages and JavaScript error types of the variants that have a C++ +/// counterpart match the corresponding `JSG_REQUIRE` / `JSG_FAIL_REQUIRE` in +/// `workerd::api::node::i18n::transcode` (`src/workerd/api/node/i18n.c++`), or +/// in `jsg::JsUint8Array::create` for [`TranscodeError::AllocationFailed`], +/// verbatim, so gate-on and gate-off are indistinguishable to JavaScript. +/// Do not reword them. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TranscodeError { + #[error("Source buffer is too large to transcode")] + SourceBufferTooLarge, + #[error("Buffer is too large to transcode")] + BufferTooLarge, + #[error("Expected UTF-16le length is too large to transcode")] + ExpectedUtf16LengthTooLarge, + #[error("Expected UTF-8 length is too large to transcode")] + ExpectedUtf8LengthTooLarge, + #[error("UTF-16le input size should be multiple of 2")] + OddUtf16leInput, + #[error("Expected UTF16 length mismatch")] + Utf16LengthMismatch, + #[error("Expected UTF8 length mismatch")] + Utf8LengthMismatch, + #[error("Unable to transcode buffer")] + UnableToTranscode, + // These two match `i18n::Converter` in `i18n.c++`. + #[error("Failed to initialize converter")] + ConverterOpenFailed, + #[error("Setting ICU substitute characters failed")] + SetSubstituteCharsFailed, + #[error("Invalid encoding passed to transcode")] + InvalidEncoding, + #[error("Failed to allocate memory for Uint8Array")] + AllocationFailed, + // Reports a broken internal invariant rather than bad input, and so has no + // C++ counterpart to match. + #[error("Destination buffer size does not match the prepared transcode")] + DestinationSizeMismatch, +} + +impl From for jsg::Error { + fn from(value: TranscodeError) -> Self { + match value { + // A `RangeError`, matching `JSG_REQUIRE(..., RangeError, ...)` in + // `jsg::JsUint8Array::create`. + TranscodeError::AllocationFailed => Self::new_range_error(value.to_string()), + // Plain JS `Error`s, matching the `JSG_REQUIRE(..., Error, ...)` + // calls in `i18n.c++`. + _ => Self::new_error(value.to_string()), + } + } +} + +#[cfg(test)] +#[path = "error-test.rs"] +mod tests; diff --git a/src/rust/i18n/ffi-test.rs b/src/rust/i18n/ffi-test.rs new file mode 100644 index 00000000000..a59c8376fa8 --- /dev/null +++ b/src/rust/i18n/ffi-test.rs @@ -0,0 +1,238 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use jsg_test::Harness; + +use super::*; + +// `Harness::new()` initializes the V8 platform, which is what installs the +// embedded ICU data. ICU converter opens fail without it, even though these +// tests never create an isolate or run JavaScript. +fn init_icu() -> Harness { + Harness::new() +} + +fn utf16le(text: &str) -> Vec { + text.encode_utf16().flat_map(u16::to_le_bytes).collect() +} + +/// Opens a converter for `to` with the `?` substitute `dispatch` configures. +fn substituting_converter(to: Encoding) -> Converter { + let converter = Converter::open(to).unwrap(); + converter + .set_subst_chars(&"?".repeat(converter.min_char_size())) + .unwrap(); + converter +} + +/// Runs [`convert_ex`] the way `dispatch` does, and returns what it wrote. +fn convert(from: Encoding, to: Encoding, source: &[u8]) -> Vec { + let to = substituting_converter(to); + let from = Converter::open(from).unwrap(); + let mut target = vec![0; source.len() * to.max_char_size()]; + let written = convert_ex(&to, &from, source, &mut target).unwrap(); + target.truncate(written); + target +} + +/// Runs [`from_uchars`] the way `dispatch` does, and returns what it wrote. +fn from_utf16le(to: Encoding, source: &[u8]) -> Vec { + let to = substituting_converter(to); + let mut target = vec![0; source.len() / 2 * to.max_char_size()]; + let written = from_uchars(&to, source, &mut target).unwrap(); + target.truncate(written); + target +} + +// `src/workerd/api/node/i18n-test.c++` checks these conversions against the +// C++ path; these pin down the ICU behaviour that path relies on. + +#[test] +fn char_sizes_come_from_icu() { + let _harness = init_icu(); + for (encoding, min, max) in [ + (Encoding::Ascii, 1, 1), + (Encoding::Latin1, 1, 1), + (Encoding::Utf8, 1, 3), + (Encoding::Utf16Le, 2, 2), + ] { + let converter = Converter::open(encoding).unwrap(); + assert_eq!(converter.min_char_size(), min, "{encoding:?}"); + assert_eq!(converter.max_char_size(), max, "{encoding:?}"); + } +} + +#[test] +fn ascii_and_latin1_are_windows_1252() { + // Chromium's ICU makes `us-ascii` and `iso8859-1` aliases of + // windows-1252. + let _harness = init_icu(); + let euro = "\u{20ac}".as_bytes(); + for encoding in [Encoding::Ascii, Encoding::Latin1] { + assert_eq!(convert(encoding, Encoding::Utf8, &[0x80]), euro); + assert_eq!( + convert(encoding, Encoding::Utf8, &[0x81]), + "\u{81}".as_bytes() + ); + assert_eq!(convert(Encoding::Utf8, encoding, euro), [0x80]); + assert_eq!( + convert(encoding, encoding, &[0x61, 0x80, 0xff]), + [0x61, 0x80, 0xff] + ); + } +} + +#[test] +fn ill_formed_input_decodes_to_replacement_characters() { + const FFFD: &[u8] = "\u{fffd}".as_bytes(); + let _harness = init_icu(); + assert_eq!( + convert(Encoding::Utf8, Encoding::Utf8, &[0xe2, 0x82, 0x41]), + [FFFD, b"A"].concat() + ); + assert_eq!( + convert( + Encoding::Utf16Le, + Encoding::Utf16Le, + &[0x00, 0xd8, 0x41, 0x00] + ), + [0xfd, 0xff, 0x41, 0x00] + ); +} + +#[test] +fn unrepresentable_characters_encode_as_question_marks() { + let _harness = init_icu(); + assert_eq!( + convert(Encoding::Utf8, Encoding::Latin1, "Ć©ā˜•".as_bytes()), + [0xe9, b'?'] + ); + // A supplementary character is one character, so one `?`. + assert_eq!( + convert(Encoding::Utf8, Encoding::Ascii, "šŸ˜€".as_bytes()), + b"?" + ); + assert_eq!(from_utf16le(Encoding::Ascii, &utf16le("šŸ˜€")), b"?"); + assert_eq!( + from_utf16le(Encoding::Latin1, &utf16le("Ć©ā˜•")), + [0xe9, b'?'] + ); +} + +#[test] +fn unrepresentable_default_ignorables_are_dropped() { + let _harness = init_icu(); + assert_eq!( + convert( + Encoding::Utf8, + Encoding::Latin1, + "a\u{200b}b\u{feff}".as_bytes() + ), + b"ab" + ); + assert_eq!(from_utf16le(Encoding::Ascii, &utf16le("a\u{200b}b")), b"ab"); +} + +#[test] +fn a_short_target_is_a_failure() { + let _harness = init_icu(); + let to = substituting_converter(Encoding::Utf8); + let from = Converter::open(Encoding::Latin1).unwrap(); + let mut target = [0; 2]; + assert_eq!(convert_ex(&to, &from, &[0xe9, 0xe9], &mut target), None); + + let to = substituting_converter(Encoding::Latin1); + let mut target = [0; 1]; + assert_eq!(from_uchars(&to, &utf16le("ab"), &mut target), None); +} + +// simdutf. `src/workerd/api/node/i18n-test.c++` checks these conversions +// against the C++ path; these pin down the behaviour `dispatch` relies on. + +#[test] +fn latin1_widens_every_byte() { + let source: Vec = (0..=255).collect(); + let mut target = vec![0; source.len() * 2]; + assert_eq!(convert_latin1_to_utf16(&source, &mut target), Some(256)); + let expected: Vec = (0..=255u16).flat_map(u16::to_le_bytes).collect(); + assert_eq!(target, expected); +} + +#[test] +fn utf16_length_from_utf8_counts_lead_bytes() { + // Exact on valid input: 1 + 1 + 1 + 2 units. + assert_eq!(Utf8Source::new("aĆ©ā˜•šŸ˜€".as_bytes()).utf16_len(), 5); + // Continuation bytes alone count nothing. + assert_eq!(Utf8Source::new(&[0x80, 0xbf]).utf16_len(), 0); +} + +#[test] +fn utf8_length_from_utf16le_is_exact_for_valid_input() { + // 1 + 2 + 3 + 4 bytes. + assert_eq!(Utf16LeSource::new(&utf16le("aĆ©ā˜•šŸ˜€")).utf8_len(), 10); + // A trailing odd byte is ignored. + assert_eq!(Utf16LeSource::new(&[0x61, 0x00, 0x62]).utf8_len(), 1); +} + +#[test] +fn utf8_to_utf16le_round_trips_valid_input() { + let text = "aĆ©ā˜•šŸ˜€"; + let utf8 = Utf8Source::new(text.as_bytes()); + let mut target = vec![0; utf8.utf16_len() * 2]; + assert_eq!(utf8.convert_to_utf16le(&mut target), Some(5)); + assert_eq!(target, utf16le(text)); +} + +#[test] +fn utf8_to_utf16le_rejects_invalid_input() { + for invalid in [ + &[0x61, 0xc3][..], // truncated + &[0xc0, 0x80], // overlong + &[0xed, 0xa0, 0x80], // encoded surrogate + &[0xf4, 0x90, 0x80, 0x80], // above U+10FFFF + &[0x61, 0x80], // stray continuation byte + ] { + let utf8 = Utf8Source::new(invalid); + let mut target = vec![0; utf8.utf16_len() * 2]; + assert_eq!( + utf8.convert_to_utf16le(&mut target), + Some(0), + "{invalid:x?}" + ); + } +} + +#[test] +fn utf16le_to_utf8_round_trips_valid_input() { + let source = utf16le("aĆ©ā˜•šŸ˜€"); + let utf16 = Utf16LeSource::new(&source); + let mut target = vec![0; utf16.utf8_len()]; + assert_eq!(utf16.convert_to_utf8(&mut target), Some(10)); + assert_eq!(target, "aĆ©ā˜•šŸ˜€".as_bytes()); +} + +#[test] +fn utf16le_to_utf8_rejects_unpaired_surrogates() { + for invalid in [ + &[0x00, 0xd8][..], // lone high surrogate + &[0x00, 0xdc], // lone low surrogate + &[0x00, 0xd8, 0x61, 0x00], // high surrogate then non-surrogate + &[0x61, 0x00, 0x00, 0xdc], // low surrogate after non-surrogate + ] { + let utf16 = Utf16LeSource::new(invalid); + let mut target = vec![0; utf16.utf8_len()]; + assert_eq!(utf16.convert_to_utf8(&mut target), Some(0), "{invalid:x?}"); + } +} + +#[test] +fn simdutf_conversions_refuse_a_target_shorter_than_the_estimate() { + let mut target = [0; 2]; + assert_eq!(convert_latin1_to_utf16(b"ab", &mut target), None); + assert_eq!(Utf8Source::new(b"ab").convert_to_utf16le(&mut target), None); + assert_eq!( + Utf16LeSource::new(&utf16le("aā˜•")).convert_to_utf8(&mut target), + None + ); +} diff --git a/src/rust/i18n/ffi.h b/src/rust/i18n/ffi.h new file mode 100644 index 00000000000..c9f20b659b7 --- /dev/null +++ b/src/rust/i18n/ffi.h @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include "simdutf.h" + +#include +#include + +// The simdutf calls `ffi.rs` makes, forwarding to the same C++ functions `i18n.c++` calls. +// simdutf's C API (`simdutf_c.h`) would avoid this shim, but the simdutf some builds link predates +// it; the C++ functions below are present in every version we link. +// +// cxx has no `char` or `char16_t`, so the bridge passes `uint8_t` and `uint16_t` pointers. The +// UTF-16 pointers need not be aligned, exactly as in `i18n.c++`. + +namespace workerd::rust::i18n { + +inline size_t simdutf_convert_latin1_to_utf16( + const uint8_t* input, size_t length, uint16_t* utf16_output) { + return simdutf::convert_latin1_to_utf16( + reinterpret_cast(input), length, reinterpret_cast(utf16_output)); +} + +inline size_t simdutf_utf16_length_from_utf8(const uint8_t* input, size_t length) { + return simdutf::utf16_length_from_utf8(reinterpret_cast(input), length); +} + +inline size_t simdutf_convert_utf8_to_utf16le( + const uint8_t* input, size_t length, uint16_t* utf16_output) { + return simdutf::convert_utf8_to_utf16le( + reinterpret_cast(input), length, reinterpret_cast(utf16_output)); +} + +inline size_t simdutf_utf8_length_from_utf16le(const uint16_t* input, size_t length) { + return simdutf::utf8_length_from_utf16le(reinterpret_cast(input), length); +} + +inline size_t simdutf_convert_utf16le_to_utf8( + const uint16_t* input, size_t length, uint8_t* utf8_output) { + return simdutf::convert_utf16le_to_utf8( + reinterpret_cast(input), length, reinterpret_cast(utf8_output)); +} + +} // namespace workerd::rust::i18n diff --git a/src/rust/i18n/ffi.rs b/src/rust/i18n/ffi.rs new file mode 100644 index 00000000000..4cf844a9b3b --- /dev/null +++ b/src/rust/i18n/ffi.rs @@ -0,0 +1,434 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The FFI island of the crate, and the only module that needs `unsafe`. It +//! holds the `#[cxx::bridge]`; the entry point that turns C++'s raw +//! `v8::Isolate*` into a [`jsg::Lock`]; and safe wrappers around the ICU +//! converter calls and simdutf calls `i18n.c++` makes, through `rust_icu_sys` +//! and `ffi.h`. Both run in the libraries workerd links, so every conversion is +//! the same code the C++ path runs. +#![allow( + unsafe_code, + reason = "the crate's FFI island: the cxx bridge, the raw-isolate entry point, and the ICU and simdutf calls" +)] + +use std::ffi::CStr; +use std::ffi::c_char; + +pub use bridge::Encoding; +use jsg::v8; +use rust_icu_sys as sys; +use rust_icu_sys::versioned_function; + +use crate::error::TranscodeError; + +#[cxx::bridge(namespace = "workerd::rust::i18n")] +mod bridge { + /// The four encodings `i18n::transcode` supports. Mirrors the + /// transcodable subset of `workerd::api::node::Encoding` + /// (`src/workerd/api/node/i18n.h`). `src/workerd/api/node/i18n.c++` maps + /// into this type through a `fromImpl` overload that rejects the + /// non-transcodable `BASE64`, `BASE64URL`, and `HEX` variants before ever + /// calling into Rust. + #[derive(Debug, PartialEq, Eq, Copy, Clone)] + #[repr(u8)] + enum Encoding { + Ascii, + Latin1, + Utf8, + Utf16Le, + } + + #[namespace = "workerd::rust::jsg"] + unsafe extern "C++" { + include!("workerd/rust/jsg/ffi.h"); + include!("workerd/rust/jsg/v8.rs.h"); + + type Isolate = jsg::v8::ffi::Isolate; + type MaybeLocal = jsg::v8::ffi::MaybeLocal; + } + + unsafe extern "C++" { + include!("workerd/rust/i18n/ffi.h"); + + unsafe fn simdutf_convert_latin1_to_utf16( + input: *const u8, + length: usize, + utf16_output: *mut u16, + ) -> usize; + unsafe fn simdutf_utf16_length_from_utf8(input: *const u8, length: usize) -> usize; + unsafe fn simdutf_convert_utf8_to_utf16le( + input: *const u8, + length: usize, + utf16_output: *mut u16, + ) -> usize; + unsafe fn simdutf_utf8_length_from_utf16le(input: *const u16, length: usize) -> usize; + unsafe fn simdutf_convert_utf16le_to_utf8( + input: *const u16, + length: usize, + utf8_output: *mut u8, + ) -> usize; + } + + extern "Rust" { + /// Transcodes `source` from `from_encoding` to `to_encoding`, matching + /// `workerd::api::node::i18n::transcode`. Returns a `MaybeLocal` + /// naming a `Uint8Array`, or an empty `MaybeLocal` with a JS exception + /// already scheduled on `isolate` if transcoding fails. + /// + /// # Safety + /// `isolate` must be a valid pointer to a locked `v8::Isolate` with an + /// active `HandleScope`. + unsafe fn transcode( + isolate: *mut Isolate, + source: &[u8], + from_encoding: Encoding, + to_encoding: Encoding, + ) -> MaybeLocal; + } +} + +/// # Safety +/// `isolate` must be a valid pointer to a locked `v8::Isolate` with an active +/// `HandleScope`. +unsafe fn transcode( + isolate: *mut bridge::Isolate, + source: &[u8], + from_encoding: Encoding, + to_encoding: Encoding, +) -> bridge::MaybeLocal { + // SAFETY: forwarded from this function's own safety contract. The adapter + // owns the raw-isolate, exception, and local-handle FFI transitions; the + // feature callback operates only on safe JSG types. + unsafe { + v8::run_ffi_callback(isolate, |lock| { + crate::transcode_impl(lock, source, from_encoding, to_encoding) + }) + } +} + +// ICU +// +// Everything unsafe about calling ICU: deriving pointers and lengths from +// slices, owning the `UConverter`, and turning ICU's out-parameter +// `UErrorCode` into `Option` and `Result`. + +/// ICU treats positive codes as failures and negative codes as warnings, +/// matching the `U_FAILURE` macro. +fn is_failure(error: sys::UErrorCode) -> bool { + error > sys::UErrorCode::U_ZERO_ERROR +} + +/// The ICU converter name for `encoding`, matching `getEncodingName()` in +/// `i18n.c++`. +/// +/// The bridge `Encoding` is a `cxx` shared enum, and so a `u8` newtype rather +/// than a real Rust enum. As in `i18n.c++`, whose `getEncodingName` ends in +/// `default: KJ_UNREACHABLE`, an unknown encoding is unreachable here: +/// `dispatch` rejects an unknown source encoding up front, the C++ `fromImpl` +/// conversion rejects anything but the four transcodable encodings before +/// Rust is entered, and a panic in Rust reaches C++ as a `kj::Exception`. +fn icu_name(encoding: Encoding) -> &'static CStr { + match encoding { + Encoding::Ascii => c"us-ascii", + Encoding::Latin1 => c"iso8859-1", + Encoding::Utf8 => c"utf-8", + Encoding::Utf16Le => c"utf16le", + _ => unreachable!("invalid encoding {encoding:?}"), + } +} + +/// An open ICU converter, mirroring `i18n::Converter` in `i18n.c++`. +/// +/// Owns its `UConverter` and closes it on drop, including while unwinding. +/// Holding a raw pointer makes the type neither `Send` nor `Sync`, which is +/// what we want: ICU converters carry conversion state and are not safe to +/// share between threads. +pub struct Converter { + cnv: *mut sys::UConverter, +} + +impl Converter { + /// Opens an ICU converter for `encoding`. + pub fn open(encoding: Encoding) -> Result { + let mut err = sys::UErrorCode::U_ZERO_ERROR; + // SAFETY: the name is a NUL-terminated static string, and `err` is a + // live local for the duration of the call. + let cnv = + unsafe { versioned_function!(ucnv_open)(icu_name(encoding).as_ptr(), &raw mut err) }; + if is_failure(err) || cnv.is_null() { + return Err(TranscodeError::ConverterOpenFailed); + } + Ok(Self { cnv }) + } + + /// The most bytes one UTF-16 code unit can occupy in this converter's + /// encoding, from `ucnv_getMaxCharSize`. + pub fn max_char_size(&self) -> usize { + // SAFETY: `self.cnv` is a live converter returned by `ucnv_open`. + let size = unsafe { versioned_function!(ucnv_getMaxCharSize)(self.cnv) }; + // ICU reports a small positive `int8_t`. + usize::from(size.unsigned_abs()) + } + + /// The fewest bytes one character can occupy in this converter's + /// encoding, from `ucnv_getMinCharSize`. + pub fn min_char_size(&self) -> usize { + // SAFETY: `self.cnv` is a live converter returned by `ucnv_open`. + let size = unsafe { versioned_function!(ucnv_getMinCharSize)(self.cnv) }; + // ICU reports a small positive `int8_t`. + usize::from(size.unsigned_abs()) + } + + /// Sets the converter's substitute character sequence, used in place of + /// unmappable characters during conversion. An empty `substitute` leaves + /// ICU's default in place, as in `i18n.c++`. + pub fn set_subst_chars(&self, substitute: &str) -> Result<(), TranscodeError> { + if substitute.is_empty() { + return Ok(()); + } + // ICU takes the length as an `int8_t` and reads a negative length as + // "NUL-terminated", which `substitute` is not. Its own limit on + // substitute sequences is far lower still. + let length = + i8::try_from(substitute.len()).map_err(|_| TranscodeError::SetSubstituteCharsFailed)?; + + let mut err = sys::UErrorCode::U_ZERO_ERROR; + // SAFETY: `self.cnv` is a live converter, and `substitute` outlives the + // call and is `length` bytes long. ICU takes the sequence as bytes and + // does not require NUL termination when given an explicit length. + unsafe { + versioned_function!(ucnv_setSubstChars)( + self.cnv, + substitute.as_ptr().cast(), + length, + &raw mut err, + ); + } + if is_failure(err) { + return Err(TranscodeError::SetSubstituteCharsFailed); + } + Ok(()) + } +} + +impl Drop for Converter { + fn drop(&mut self) { + // SAFETY: `self.cnv` was returned non-null by `ucnv_open` and is closed + // exactly once, here. + unsafe { versioned_function!(ucnv_close)(self.cnv) } + } +} + +/// Converts `source` from `from`'s encoding to `to`'s via ICU's +/// `ucnv_convertEx`, as `TranscodeDefault` in `i18n.c++` does. Returns the +/// number of bytes written to `target`, or `None` if ICU reports failure. +#[must_use] +pub fn convert_ex( + to: &Converter, + from: &Converter, + source: &[u8], + target: &mut [u8], +) -> Option { + let target_start: *mut c_char = target.as_mut_ptr().cast(); + let mut target_cursor = target_start; + let mut source_cursor: *const c_char = source.as_ptr().cast(); + let mut err = sys::UErrorCode::U_ZERO_ERROR; + + // SAFETY: both cursors start at the base of a live slice and are bounded + // by a limit one past that slice's end, which is what ICU advances them + // against. Passing a null pivot asks ICU to use an internal one. + unsafe { + versioned_function!(ucnv_convertEx)( + to.cnv, + from.cnv, + &raw mut target_cursor, + target_start.add(target.len()), + &raw mut source_cursor, + source_cursor.add(source.len()), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null(), + 1, // reset + 1, // flush + &raw mut err, + ); + } + if is_failure(err) { + return None; + } + // SAFETY: ICU advanced `target_cursor` within `target`, so both pointers + // are into the same allocation. + let written = unsafe { target_cursor.offset_from(target_start) }; + usize::try_from(written).ok() +} + +/// Converts UTF-16LE `source` (as raw bytes) to `to`'s encoding via ICU's +/// `ucnv_fromUChars`, as `TranscodeFromUTF16` in `i18n.c++` does. A trailing +/// odd byte is ignored. Returns the number of bytes written to `target`, or +/// `None` if ICU reports failure. +#[must_use] +pub fn from_uchars(to: &Converter, source: &[u8], target: &mut [u8]) -> Option { + let src_length = i32::try_from(source.len() / size_of::()).ok()?; + let dest_capacity = i32::try_from(target.len()).ok()?; + let mut err = sys::UErrorCode::U_ZERO_ERROR; + + // SAFETY: the pointers and lengths describe the two live slices. `source` + // need not be `u16`-aligned -- it is caller-supplied buffer contents, which + // a Uint8Array can expose at an odd byteOffset -- and is reinterpreted as + // `UChar*` exactly as `i18n.c++` does with the same bytes. Casting a raw + // pointer is well-defined in Rust regardless of alignment; no reference to + // the misaligned data is ever formed on this side. + let len = unsafe { + versioned_function!(ucnv_fromUChars)( + to.cnv, + target.as_mut_ptr().cast(), + dest_capacity, + source.as_ptr().cast(), + src_length, + &raw mut err, + ) + }; + if is_failure(err) { + return None; + } + usize::try_from(len).ok() +} + +// simdutf +// +// The simdutf C++ functions `i18n.c++` calls, through the inline forwarders in +// `ffi.h`, declared in the bridge above. +// +// simdutf does not bound its writes by a destination length. Its conversions +// write at most the length its own estimate reports for the same input, valid +// or not: its SIMD kernels stop short of the end of the input by a margin +// that keeps their wide stores inside an estimate-sized buffer ("to avoid +// overruns", in `utf8_to_utf16::validating_transcoder::convert` and +// `avx2_convert_utf16_to_utf8`), and report errors only after converting. The +// C++ path relies on this too. [`Utf8Source`] and [`Utf16LeSource`] hold each +// source together with the estimate taken from it, so a conversion can check +// its destination against that estimate. +// +// UTF-16 is passed to simdutf as raw bytes reinterpreted as `char16_t*`, +// exactly as `i18n.c++` passes the same bytes, and so need not be +// `u16`-aligned; see [`from_uchars`]. + +/// Widens Latin-1 `source` into UTF-16LE (written to `target` as raw bytes) +/// via `simdutf::convert_latin1_to_utf16`, as `TranscodeLatin1ToUTF16` in +/// `i18n.c++` does. Returns the number of UTF-16 code units written, or `None` +/// if `target` cannot hold two bytes per source byte. +#[must_use] +pub fn convert_latin1_to_utf16(source: &[u8], target: &mut [u8]) -> Option { + if target.len() / size_of::() < source.len() { + return None; + } + // SAFETY: every Latin-1 byte widens to exactly one code unit, and `target` + // holds at least `source.len()` of them. + Some(unsafe { + bridge::simdutf_convert_latin1_to_utf16( + source.as_ptr().cast(), + source.len(), + target.as_mut_ptr().cast(), + ) + }) +} + +/// UTF-8 bytes measured by `simdutf::utf16_length_from_utf8`. +pub struct Utf8Source<'a> { + bytes: &'a [u8], + utf16_len: usize, +} + +impl<'a> Utf8Source<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + // SAFETY: pointer and length describe a live slice. + let utf16_len = + unsafe { bridge::simdutf_utf16_length_from_utf8(bytes.as_ptr().cast(), bytes.len()) }; + Self { bytes, utf16_len } + } + + /// simdutf's estimate of the UTF-16 length, in code units. Exact for + /// valid UTF-8; for invalid UTF-8 it is only the count simdutf reports. + pub const fn utf16_len(&self) -> usize { + self.utf16_len + } + + /// Converts to UTF-16LE (written to `target` as raw bytes) via + /// `simdutf::convert_utf8_to_utf16le`, as `TranscodeUTF16FromUTF8` in + /// `i18n.c++` does. Returns the number of code units written, which is `0` + /// for invalid UTF-8, or `None` if `target` cannot hold + /// [`Utf8Source::utf16_len`] code units. + pub fn convert_to_utf16le(&self, target: &mut [u8]) -> Option { + if target.len() / 2 < self.utf16_len { + return None; + } + // SAFETY: `target` holds the `utf16_len` code units simdutf estimated + // for these same bytes, which is the most it writes for them; see the + // section comment above. + Some(unsafe { + bridge::simdutf_convert_utf8_to_utf16le( + self.bytes.as_ptr().cast(), + self.bytes.len(), + target.as_mut_ptr().cast(), + ) + }) + } +} + +/// UTF-16LE code units (as raw bytes) measured by +/// `simdutf::utf8_length_from_utf16le`. A trailing odd byte is ignored. +pub struct Utf16LeSource<'a> { + bytes: &'a [u8], + utf8_len: usize, +} + +impl<'a> Utf16LeSource<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + // SAFETY: pointer and length describe the whole code units of a live + // slice. + let utf8_len = unsafe { + bridge::simdutf_utf8_length_from_utf16le( + bytes.as_ptr().cast(), + bytes.len() / size_of::(), + ) + }; + Self { bytes, utf8_len } + } + + /// simdutf's estimate of the UTF-8 length, in bytes. Exact for valid + /// UTF-16. An unpaired surrogate has no UTF-8 encoding, and what it counts + /// for depends on the kernel simdutf selects for the CPU: two bytes, or + /// three (the size of U+FFFD) in the AVX-512 kernel's path for inputs of + /// up to 32 code units. + pub const fn utf8_len(&self) -> usize { + self.utf8_len + } + + /// Converts to UTF-8 via `simdutf::convert_utf16le_to_utf8`, as + /// `TranscodeUTF8FromUTF16` in `i18n.c++` does. Returns the number of + /// bytes written, which is `0` if the source contains an unpaired + /// surrogate, or `None` if `target` cannot hold + /// [`Utf16LeSource::utf8_len`] bytes. + pub fn convert_to_utf8(&self, target: &mut [u8]) -> Option { + if target.len() < self.utf8_len { + return None; + } + // SAFETY: `target` holds the `utf8_len` bytes simdutf estimated for + // these same code units, which is the most it writes for them; see + // the section comment above. + Some(unsafe { + bridge::simdutf_convert_utf16le_to_utf8( + self.bytes.as_ptr().cast(), + self.bytes.len() / size_of::(), + target.as_mut_ptr().cast(), + ) + }) + } +} + +#[cfg(test)] +#[path = "ffi-test.rs"] +mod tests; diff --git a/src/rust/i18n/lib-test.rs b/src/rust/i18n/lib-test.rs new file mode 100644 index 00000000000..caa8b1dde5b --- /dev/null +++ b/src/rust/i18n/lib-test.rs @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use jsg_test::Harness; + +use super::*; + +/// Exercises the full V8 path: allocate, convert into the backing store, +/// narrow the view. `dispatch-test.rs` covers conversion behaviour itself; this +/// checks the parts that only exist once V8 is involved. +#[test] +fn transcodes_into_a_narrowed_uint8_array() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // 'ā˜•' is three UTF-8 bytes and transcodes to the single byte "?" + // in ASCII, so the destination is allocated at 3 bytes and the + // returned view must be narrowed to 1. + let source = "ā˜•".as_bytes(); + let array = + transcode_impl(lock, source, ffi::Encoding::Utf8, ffi::Encoding::Ascii).unwrap(); + assert_eq!(array.len(), 1); + assert_eq!(array.as_slice(), b"?"); + Ok(()) + }); +} + +#[test] +fn empty_input_yields_an_empty_uint8_array() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + let array = transcode_impl(lock, &[], ffi::Encoding::Utf8, ffi::Encoding::Utf16Le).unwrap(); + assert!(array.is_empty()); + Ok(()) + }); +} + +#[test] +fn full_length_result_is_not_narrowed() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // Latin-1 -> UTF-16LE widens every byte to exactly two, so the + // conversion fills the destination exactly. + let array = + transcode_impl(lock, b"Hi", ffi::Encoding::Latin1, ffi::Encoding::Utf16Le).unwrap(); + assert_eq!(array.as_slice(), &[0x48, 0x00, 0x69, 0x00]); + Ok(()) + }); +} + +#[test] +fn failure_surfaces_as_an_error() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // Odd-length UTF-16LE input is rejected before any allocation. + let result = transcode_impl(lock, &[0x61], ffi::Encoding::Utf16Le, ffi::Encoding::Utf8); + assert!(result.is_err()); + Ok(()) + }); +} diff --git a/src/rust/i18n/lib.rs b/src/rust/i18n/lib.rs new file mode 100644 index 00000000000..6fe4c7d8e2d --- /dev/null +++ b/src/rust/i18n/lib.rs @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Rust port of `workerd::api::node::i18n::transcode` +//! (`src/workerd/api/node/i18n.c++`), the engine behind `node:buffer`'s +//! `transcode()`. Selected at runtime by the `NODEJS_I18N_RUST` autogate; when +//! the gate is off, the C++ implementation is used instead. The two paths are +//! byte-for-byte and error-message identical: [`dispatch`] ports the C++ +//! dispatch/sizing/truncation logic to Rust, and [`ffi`] calls the same ICU +//! and simdutf functions the C++ path does. +//! `src/workerd/api/node/i18n-test.c++` checks the two paths against each +//! other. + +use jsg::Lock; +use jsg::v8; + +mod dispatch; +mod error; +mod ffi; + +use crate::dispatch::Transcoder; +use crate::error::TranscodeError; + +/// Transcodes `source` into a freshly allocated `Uint8Array`. +/// +/// The conversion writes straight into the V8 `ArrayBuffer`'s backing store. +/// There is no intermediate `Vec` and no copy: [`Transcoder`] reports the +/// destination size up front, the buffer is allocated at exactly that size, +/// and the returned view is narrowed to the bytes actually written. +fn transcode_impl<'a>( + lock: &'a mut Lock, + source: &[u8], + from_encoding: ffi::Encoding, + to_encoding: ffi::Encoding, +) -> jsg::Result> { + let transcoder = Transcoder::new(source, from_encoding, to_encoding)?; + let (buffer, written) = v8::ArrayBuffer::new_zeroed_with(lock, transcoder.dest_len(), |dest| { + transcoder.transcode_into(dest) + }) + .ok_or(TranscodeError::AllocationFailed)?; + v8::Uint8Array::from_buffer(lock, &buffer, 0, written?) +} + +#[cfg(test)] +#[path = "lib-test.rs"] +mod tests; diff --git a/src/rust/jsg-test/tests/buffer_types.rs b/src/rust/jsg-test/tests/buffer_types.rs index 0fa0ede2a2f..6cbad6bf7a1 100644 --- a/src/rust/jsg-test/tests/buffer_types.rs +++ b/src/rust/jsg-test/tests/buffer_types.rs @@ -62,6 +62,79 @@ fn array_buffer_new_zeroed_is_zeroed() { }); } +#[test] +fn array_buffer_new_zeroed_with_fills_before_js_sees_it() { + let harness = crate::Harness::new(); + harness.run_in_context(|lock, ctx| { + let (buf, result) = v8::ArrayBuffer::new_zeroed_with(lock, 4, |bytes| { + assert_eq!(bytes, &[0, 0, 0, 0], "must be zeroed before fill runs"); + bytes[1..3].copy_from_slice(&[10, 20]); + "done" + }) + .unwrap(); + assert_eq!(result, "done"); + assert_eq!(buf.as_slice(), &[0, 10, 20, 0]); + + // The written bytes are visible to JavaScript and the rest stay zero. + ctx.set_global("buf", buf.into()); + let contents: String = ctx.eval(lock, "new Uint8Array(buf).join(',')").unwrap(); + assert_eq!(contents, "0,10,20,0"); + + Ok(()) + }); +} + +#[test] +fn array_buffer_new_zeroed_with_empty() { + let harness = crate::Harness::new(); + harness.run_in_context(|lock, _ctx| { + let (buf, len) = v8::ArrayBuffer::new_zeroed_with(lock, 0, |bytes| bytes.len()).unwrap(); + assert_eq!(len, 0); + assert!(buf.is_empty()); + Ok(()) + }); +} + +#[test] +fn array_buffer_new_zeroed_with_allocation_failure_skips_fill() { + let harness = crate::Harness::new(); + harness.run_in_context(|lock, _ctx| { + let mut called = false; + let result = v8::ArrayBuffer::new_zeroed_with(lock, usize::MAX, |_| called = true); + assert!(result.is_none()); + assert!(!called); + Ok(()) + }); +} + +#[test] +fn uint8_array_from_buffer_views_a_range() { + let harness = crate::Harness::new(); + harness.run_in_context(|lock, _ctx| { + let buf = v8::ArrayBuffer::new(lock, &[1, 2, 3, 4]); + let view = v8::Uint8Array::from_buffer(lock, &buf, 1, 2).unwrap(); + assert_eq!(view.as_slice(), &[2, 3]); + let whole = v8::Uint8Array::from_buffer(lock, &buf, 0, 4).unwrap(); + assert_eq!(whole.as_slice(), &[1, 2, 3, 4]); + let empty_at_end = v8::Uint8Array::from_buffer(lock, &buf, 4, 0).unwrap(); + assert!(empty_at_end.is_empty()); + Ok(()) + }); +} + +#[test] +fn uint8_array_from_buffer_out_of_bounds_is_an_error() { + let harness = crate::Harness::new(); + harness.run_in_context(|lock, _ctx| { + let buf = v8::ArrayBuffer::new(lock, &[1, 2, 3, 4]); + assert!(v8::Uint8Array::from_buffer(lock, &buf, 0, 5).is_err()); + assert!(v8::Uint8Array::from_buffer(lock, &buf, 5, 0).is_err()); + assert!(v8::Uint8Array::from_buffer(lock, &buf, 3, 2).is_err()); + assert!(v8::Uint8Array::from_buffer(lock, &buf, 1, usize::MAX).is_err()); + Ok(()) + }); +} + #[test] fn array_buffer_is_empty() { let harness = crate::Harness::new(); diff --git a/src/rust/jsg/ffi-inl.h b/src/rust/jsg/ffi-inl.h index 70eb9af97bd..9cc65aee465 100644 --- a/src/rust/jsg/ffi-inl.h +++ b/src/rust/jsg/ffi-inl.h @@ -50,6 +50,12 @@ inline MaybeLocal maybe_local_to_ffi(v8::MaybeLocal value) { return MaybeLocal{result}; } +template +inline v8::MaybeLocal maybe_local_from_ffi(MaybeLocal&& value) { + auto ptr_void = reinterpret_cast(&value.ptr); + return *reinterpret_cast*>(ptr_void); +} + // Global // // ffi::Global stores only the strong v8::Global in `ptr`. diff --git a/src/rust/jsg/ffi.c++ b/src/rust/jsg/ffi.c++ index b8b870478cf..ae38144c0a8 100644 --- a/src/rust/jsg/ffi.c++ +++ b/src/rust/jsg/ffi.c++ @@ -788,6 +788,15 @@ DEFINE_TYPED_ARRAY_UNWRAP(biguint64_array, BigUint64Array, uint64_t) } // Local +Local uint8_array_from_buffer( + Isolate* isolate, const Local& buffer, size_t byte_offset, size_t length) { + auto arrayBuffer = local_as_ref_from_ffi(buffer); + KJ_REQUIRE( + byte_offset <= arrayBuffer->ByteLength() && length <= arrayBuffer->ByteLength() - byte_offset, + "Uint8Array view is out of bounds of its ArrayBuffer"); + return to_ffi(v8::Uint8Array::New(arrayBuffer, byte_offset, length)); +} + size_t local_typed_array_length(Isolate* isolate, const Local& array) { return local_as_ref_from_ffi(array)->Length(); } diff --git a/src/rust/jsg/ffi.h b/src/rust/jsg/ffi.h index 989a7e0de63..aacdffc1a28 100644 --- a/src/rust/jsg/ffi.h +++ b/src/rust/jsg/ffi.h @@ -207,6 +207,13 @@ Local local_new_float32_array(Isolate* isolate, const float* data, size_t length Local local_new_float64_array(Isolate* isolate, const double* data, size_t length); Local local_new_bigint64_array(Isolate* isolate, const int64_t* data, size_t length); Local local_new_biguint64_array(Isolate* isolate, const uint64_t* data, size_t length); +// Creates a Uint8Array view over an existing ArrayBuffer, without copying. Unlike +// local_new_uint8_array, which allocates a backing store and memcpys into it, this +// lets a caller allocate the buffer up front, write into it directly, and only then +// wrap the written prefix in a view. `byte_offset + length` must be within +// `buffer`'s byte length. +Local uint8_array_from_buffer( + Isolate* isolate, const Local& buffer, size_t byte_offset, size_t length); size_t local_typed_array_length(Isolate* isolate, const Local& array); // Returns a raw pointer to the underlying ArrayBuffer's data (without byte offset). // Use local_typed_array_byte_offset to compute the start of this view's data. diff --git a/src/rust/jsg/v8.rs b/src/rust/jsg/v8.rs index 4ae7958db18..bf21c839cda 100644 --- a/src/rust/jsg/v8.rs +++ b/src/rust/jsg/v8.rs @@ -412,6 +412,13 @@ pub mod ffi { ) -> Result>; // Local + // Fallible: KJ_REQUIREs the view is within the buffer's bounds. + pub unsafe fn uint8_array_from_buffer( + isolate: *mut Isolate, + buffer: &Local, + byte_offset: usize, + length: usize, + ) -> Result; pub unsafe fn local_typed_array_length(isolate: *mut Isolate, array: &Local) -> usize; pub unsafe fn local_typed_array_buffer_data(isolate: *mut Isolate, array: &Local) -> usize; pub unsafe fn local_typed_array_byte_offset(isolate: *mut Isolate, array: &Local) -> usize; @@ -1712,6 +1719,39 @@ impl ArrayBuffer { ) } } + + /// Attempts to create a new zero-initialized `ArrayBuffer` with + /// `byte_length` bytes, and lets `fill` write into it before anything else + /// can observe it. Returns the buffer along with `fill`'s result, or `None` + /// if allocation fails, in which case `fill` is not called. + /// + /// This is the safe way to produce an `ArrayBuffer` from bytes computed in + /// Rust without an intermediate `Vec`: unlike the `unsafe` + /// `Local::::as_mut_slice`, exclusive access to the backing + /// store is guaranteed by construction rather than by the caller. + /// + /// Bytes `fill` does not write remain zero. The buffer is always zeroed + /// first because JavaScript can read the whole backing store once it has + /// the buffer, including any bytes a view over it excludes. + pub fn new_zeroed_with<'a, R>( + lock: &mut crate::Lock, + byte_length: usize, + fill: impl FnOnce(&mut [u8]) -> R, + ) -> Option<(Local<'a, Self>, R)> { + let mut buffer = Self::new_with_mode( + lock, + byte_length, + ffi::BackingStoreInitializationMode::ZeroInitialized, + )?; + // SAFETY: `buffer` was allocated just above and is the only handle to + // it: no other `Local` refers to it and it has never been exposed to + // JavaScript, so nothing else can reach its backing store until it is + // returned. `fill` is not passed the `Lock` and cannot capture it, + // since `lock` is mutably borrowed for this call, so it cannot run + // JavaScript. The slice does not outlive `fill`. + let result = fill(unsafe { buffer.as_mut_slice(lock) }); + Some((buffer, result)) + } } impl Local<'_, ArrayBuffer> { @@ -2336,6 +2376,38 @@ impl_typed_array!(BigUint64Array, u64, local_biguint64_array_get); // Uint8ClampedArray has the same element type as Uint8Array; clamping is a write-side JS concern. impl_typed_array!(Uint8ClampedArray, u8, local_uint8clamped_array_get); +impl Uint8Array { + /// Creates a `Uint8Array` viewing `length` bytes of `buffer`, starting at + /// `byte_offset`. + /// + /// Zero-copy, unlike [`Vec::to_js`](crate::ToJS), which allocates a fresh + /// backing store and copies into it. Producing a `Uint8Array` from bytes computed + /// in Rust therefore does not require an intermediate `Vec`: allocate and fill + /// the buffer with [`ArrayBuffer::new_zeroed_with`], and wrap the written prefix + /// here. + /// + /// # Errors + /// + /// Returns an error if `byte_offset + length` exceeds `buffer`'s byte + /// length. + pub fn from_buffer<'a>( + lock: &mut crate::Lock, + buffer: &Local<'_, ArrayBuffer>, + byte_offset: usize, + length: usize, + ) -> crate::Result> { + let isolate = lock.isolate(); + // SAFETY: Lock guarantees the isolate is locked and a HandleScope is active; + // `buffer` is a live handle to an ArrayBuffer. The C++ side checks the + // bounds and reports a violation as an `Err`. + let handle = unsafe { + ffi::uint8_array_from_buffer(isolate.as_ffi(), &buffer.handle, byte_offset, length) + }?; + // SAFETY: `handle` is a fresh Uint8Array created in `isolate`. + Ok(unsafe { Local::from_ffi(isolate, handle) }) + } +} + // ============================================================================= // `String`-specific implementations // ============================================================================= @@ -2505,6 +2577,43 @@ impl<'a, T> From>> for MaybeLocal<'a, T> { } } +/// Runs a synchronous callback entered from C++ with a valid, locked V8 isolate. +/// +/// The callback receives a safe [`Lock`] and returns a local handle. On success, +/// the handle is transferred back across the CXX bridge as a [`ffi::MaybeLocal`]. +/// On failure, the error is scheduled as a JavaScript exception and an empty +/// `MaybeLocal` is returned. +/// +/// This centralizes the raw-isolate and local-handle ownership transitions for +/// C++ entry points implemented in Rust. The callback itself contains no FFI +/// safety obligations. +/// +/// # Safety +/// +/// `isolate` must point to a live V8 isolate locked by the current thread, and +/// the C++ caller must keep its active `HandleScope` alive until it consumes the +/// returned handle. +pub unsafe fn run_ffi_callback(isolate: *mut ffi::Isolate, callback: F) -> ffi::MaybeLocal +where + E: Into, + F: for<'a> FnOnce(&'a mut Lock) -> Result, E>, +{ + // SAFETY: forwarded from this function's safety contract. + let mut lock = unsafe { Lock::from_isolate_ptr(isolate) }; + let result = callback(&mut lock).map(|local| { + // SAFETY: the handle is consumed by the C++ caller while its + // `HandleScope` is still alive, per this function's safety contract. + unsafe { local.into_ffi() } + }); + match result { + Ok(local) => ffi::MaybeLocal { ptr: local.ptr }, + Err(error) => { + lock.throw_exception(&error.into()); + ffi::MaybeLocal { ptr: 0 } + } + } +} + impl Local<'_, String> { // Instance methods ā€” correspond to `v8::String` member functions diff --git a/src/workerd/api/node/BUILD.bazel b/src/workerd/api/node/BUILD.bazel index 9d2c55ae9e7..65d79b39f02 100644 --- a/src/workerd/api/node/BUILD.bazel +++ b/src/workerd/api/node/BUILD.bazel @@ -91,6 +91,7 @@ wd_cc_library( "sqlite.h", ], implementation_deps = [ + "//src/rust/i18n", "@nbytes", "@simdutf", ], @@ -105,6 +106,18 @@ wd_cc_library( ], ) +kj_test( + # Runs every corpus input through both implementations for all 16 encoding + # pairs; the Windows CI runners take about 60s. + size = "enormous", + src = "i18n-test.c++", + deps = [ + ":node-core", + "//src/workerd/jsg", + "//src/workerd/util:autogate", + ], +) + kj_test( src = "buffer-test.c++", deps = ["//src/workerd/tests:test-fixture"], diff --git a/src/workerd/api/node/i18n-test.c++ b/src/workerd/api/node/i18n-test.c++ new file mode 100644 index 00000000000..61246cc0314 --- /dev/null +++ b/src/workerd/api/node/i18n-test.c++ @@ -0,0 +1,396 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Differential test of the two `i18n::transcode` implementations: runs the same +// inputs through the C++ implementation (NODEJS_I18N_RUST off) and the Rust one +// (NODEJS_I18N_RUST on), and requires identical results -- the same backing +// store, byte for byte (JavaScript can observe all of it through the result's +// `buffer`), the same view into it, or the same error. + +#include "i18n.h" + +#include +#include +#include +#include + +#include +#include + +namespace workerd::api::node { +namespace { + +jsg::V8System v8System; + +struct I18nContext: public jsg::Object, public jsg::ContextGlobal { + JSG_RESOURCE_TYPE(I18nContext) {} +}; +JSG_DECLARE_ISOLATE_TYPE(I18nIsolate, I18nContext); + +constexpr Encoding ENCODINGS[] = { + Encoding::ASCII, Encoding::LATIN1, Encoding::UTF8, Encoding::UTF16LE}; + +// Selects which implementation `i18n::transcode` dispatches to. Ignores +// WORKERD_ALL_AUTOGATES, which would otherwise force the gate on in the +// @all-autogates variant. +void useRust(bool enabled) { + util::Autogate::deinitAutogate(); + if (enabled) { + util::Autogate::initAutogateNamesForTest( + {kj::str(util::AutogateKey::NODEJS_I18N_RUST)}, util::IgnoreAllAutogatesEnv::YES); + } else { + util::Autogate::initAutogateNamesForTest({}, util::IgnoreAllAutogatesEnv::YES); + } + KJ_ASSERT(util::Autogate::isEnabled(util::AutogateKey::NODEJS_I18N_RUST) == enabled); +} + +// Hex for short byte strings; for long ones, the length and a prefix. +kj::String describeBytes(kj::ArrayPtr bytes) { + constexpr size_t MAX_HEX = 128; + if (bytes.size() <= MAX_HEX) return kj::encodeHex(bytes); + return kj::str(bytes.size(), " bytes starting ", kj::encodeHex(bytes.first(MAX_HEX))); +} + +// Everything JavaScript can observe about one transcode: the whole backing +// store (including any bytes outside the view), the view's placement in it, or +// the exception thrown. +struct Outcome { + kj::Array buffer; + size_t byteOffset = 0; + size_t byteLength = 0; + kj::Maybe exception; + + bool operator==(const Outcome& other) const { + return buffer == other.buffer && byteOffset == other.byteOffset && + byteLength == other.byteLength && exception == other.exception; + } + + kj::String toString() const { + KJ_IF_SOME(e, exception) { + return kj::str("threw ", e); + } + return kj::str( + "buffer=", describeBytes(buffer), " byteOffset=", byteOffset, " byteLength=", byteLength); + } +}; + +kj::String describeMismatch(const Outcome& expected, const Outcome& actual) { + constexpr size_t MAX_BUFFER = 256; + if (expected.exception != kj::none || actual.exception != kj::none || + (expected.buffer.size() <= MAX_BUFFER && actual.buffer.size() <= MAX_BUFFER)) { + return kj::str("expected ", expected.toString(), "; actual ", actual.toString()); + } + size_t common = kj::min(expected.buffer.size(), actual.buffer.size()); + size_t firstDifference = 0; + while (firstDifference < common && + expected.buffer[firstDifference] == actual.buffer[firstDifference]) { + ++firstDifference; + } + constexpr size_t CONTEXT_BYTES = 16; + auto context = [&](const Outcome& outcome) { + auto start = firstDifference - kj::min(firstDifference, CONTEXT_BYTES); + auto end = kj::min(outcome.buffer.size(), firstDifference + CONTEXT_BYTES); + return kj::encodeHex(outcome.buffer.slice(start, end)); + }; + return kj::str("buffers first differ at byte ", firstDifference, "; expected ", + expected.buffer.size(), " bytes, byteOffset=", expected.byteOffset, + " byteLength=", expected.byteLength, " (", context(expected), "); actual ", + actual.buffer.size(), " bytes, byteOffset=", actual.byteOffset, + " byteLength=", actual.byteLength, " (", context(actual), ")"); +} + +Outcome transcodeOutcome( + jsg::Lock& js, kj::ArrayPtr source, Encoding from, Encoding to) { + return js.withinHandleScope([&]() -> Outcome { + // `transcode` takes a mutable view; give it a private copy. + auto copy = kj::heapArray(source); + return js.tryCatch([&]() -> Outcome { + v8::Local result = i18n::transcode(js, copy, from, to); + auto buffer = result->Buffer(); + return Outcome{ + .buffer = kj::heapArray(kj::ArrayPtr( + static_cast(buffer->Data()), buffer->ByteLength())), + .byteOffset = result->ByteOffset(), + .byteLength = result->ByteLength(), + }; + }, [&](jsg::Value exception) -> Outcome { + return Outcome{ + .exception = jsg::JsValue(exception.getHandle(js)).toString(js), + }; + }); + }); +} + +// Bytes that exercise the interesting cases of every source encoding: ASCII +// and windows-1252 boundaries; UTF-8 lead, continuation, overlong, +// surrogate, out-of-range, and never-valid bytes; and the high bytes of +// UTF-16 surrogates and noncharacters. +constexpr kj::byte INTERESTING[] = {0x00, 0x41, 0x7f, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbb, 0xbf, + 0xc0, 0xc2, 0xdf, 0xe0, 0xe2, 0xed, 0xef, 0xf0, 0xf4, 0xf5, 0xff, 0xd8, 0xdb, 0xdc, 0xfe, 0xfd}; + +// A smaller set, for exhaustive three- and four-byte inputs: enough for every +// UTF-8 sequence length and its continuation-byte bounds, and for UTF-16 +// surrogate pairs in both orders. +constexpr kj::byte SEQUENCE_BYTES[] = {0x41, 0x80, 0xbf, 0xc2, 0xe2, 0xed, 0xf0, 0xf4, 0xd8, 0xdc}; + +// A deterministic pseudo-random generator (xorshift64). +struct Random { + uint64_t state = 0x9e3779b97f4a7c15; + + uint64_t next() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; + } + + // A `length`-byte input, mostly interesting bytes so that ill-formed + // sequences are common, with some arbitrary bytes among them. + kj::Array input(size_t length) { + auto input = kj::heapArray(length); + for (auto& byte: input) { + auto r = next(); + byte = (r & 0xff) < 192 ? INTERESTING[(r >> 8) % kj::size(INTERESTING)] + : static_cast(r >> 8); + } + return input; + } +}; + +// Appends every `length`-byte string over `alphabet` to `corpus`. +void addAllStrings( + kj::Vector>& corpus, kj::ArrayPtr alphabet, size_t length) { + auto indices = kj::heapArray(length); + indices.asPtr().fill(0); + for (;;) { + corpus.add(KJ_MAP(i, indices) { return alphabet[i]; }); + size_t position = length; + while (position > 0 && indices[position - 1] + 1 == alphabet.size()) { + --position; + } + if (position == 0) return; + ++indices[position - 1]; + for (size_t i = position; i < length; ++i) indices[i] = 0; + } +} + +// Appends `codePoint` to `out` as UTF-8. +void appendUtf8(kj::Vector& out, char32_t codePoint) { + if (codePoint < 0x80) { + out.add(static_cast(codePoint)); + } else if (codePoint < 0x800) { + out.add(static_cast(0xc0 | (codePoint >> 6))); + out.add(static_cast(0x80 | (codePoint & 0x3f))); + } else if (codePoint < 0x10000) { + out.add(static_cast(0xe0 | (codePoint >> 12))); + out.add(static_cast(0x80 | ((codePoint >> 6) & 0x3f))); + out.add(static_cast(0x80 | (codePoint & 0x3f))); + } else { + out.add(static_cast(0xf0 | (codePoint >> 18))); + out.add(static_cast(0x80 | ((codePoint >> 12) & 0x3f))); + out.add(static_cast(0x80 | ((codePoint >> 6) & 0x3f))); + out.add(static_cast(0x80 | (codePoint & 0x3f))); + } +} + +void appendUtf16Unit(kj::Vector& out, char16_t unit) { + out.add(static_cast(unit & 0xff)); + out.add(static_cast(unit >> 8)); +} + +// Appends `codePoint` to `out` as UTF-16LE. +void appendUtf16le(kj::Vector& out, char32_t codePoint) { + if (codePoint < 0x10000) { + appendUtf16Unit(out, static_cast(codePoint)); + } else { + codePoint -= 0x10000; + appendUtf16Unit(out, static_cast(0xd800 | (codePoint >> 10))); + appendUtf16Unit(out, static_cast(0xdc00 | (codePoint & 0x3ff))); + } +} + +using Encoder = void (*)(kj::Vector&, char32_t); + +kj::Array encode(kj::ArrayPtr codePoints, Encoder encoder) { + kj::Vector out; + for (auto codePoint: codePoints) encoder(out, codePoint); + return out.releaseAsArray(); +} + +// Lengths around the block sizes the codecs work in: simdutf's SIMD kernels +// (16 to 64 bytes per step) and ICU's conversion pivot (1024 UChars, so 2048 +// bytes of UTF-16), plus lengths long enough to take many steps of each. +constexpr size_t LONG_LENGTHS[] = { + 15, 16, 17, 63, 64, 65, 1023, 1024, 1025, 2047, 2048, 2049, 4097, 65537}; + +// Appends inputs of several kilobytes, so that conversions cross the SIMD and +// ICU block boundaries mid-character, and so that the SIMD fast paths for +// pure-ASCII and well-formed input run over many blocks. +void addLongInputs(kj::Vector>& corpus, Random& random) { + // Characters of every UTF-8 and UTF-16 length, including a default + // ignorable and a character only windows-1252 can represent. + constexpr char32_t MIXED[] = {U'a', U'\u00e9', U'\u2615', U'\U0001F600', U'\u200B', U'\u20AC'}; + + for (Encoder encoder: {appendUtf8, appendUtf16le}) { + kj::Vector text; + for (size_t i = 0; text.size() < 65537 + 4; ++i) encoder(text, MIXED[i % kj::size(MIXED)]); + for (auto length: LONG_LENGTHS) { + // Cut at `length` bytes, which can split a character. + corpus.add(kj::heapArray(text.asPtr().first(length))); + } + + // Well-formed text, but for one ill-formed byte (UTF-8) or unpaired + // surrogate (UTF-16) at a block boundary, or at either end. + constexpr size_t POSITIONS[] = {0, 2, 62, 64, 1022, 1024, 2048, 4096, 8188, 8190}; + auto base = text.asPtr().first(8192); + for (auto position: POSITIONS) { + auto input = kj::heapArray(base); + if (encoder == appendUtf8) { + input[position] = 0xff; + } else { + input[position] = 0x00; + input[position + 1] = 0xd8; + } + corpus.add(kj::mv(input)); + } + } + + for (auto length: LONG_LENGTHS) { + // Pure ASCII. + auto ascii = kj::heapArray(length); + for (size_t i = 0; i < length; ++i) ascii[i] = static_cast(0x20 + i % 0x5f); + corpus.add(kj::mv(ascii)); + + // Only high bytes, including 0x80-0x9f, where windows-1252 departs from + // Latin-1. + auto high = kj::heapArray(length); + for (size_t i = 0; i < length; ++i) high[i] = static_cast(0x80 + i % 0x80); + corpus.add(kj::mv(high)); + } + + // Long pseudo-random inputs. + for (size_t n = 0; n < 20; ++n) { + corpus.add(random.input(static_cast(random.next() % 20000) + 1000)); + } +} + +// Supplementary code points at the edges of the ranges ICU drops, rather than +// substitutes, when a converter cannot represent them, along with the +// largest code point. +constexpr char32_t SUPPLEMENTARY_EDGES[] = {0x1bc9f, 0x1bca0, 0x1bca3, 0x1bca4, 0x1d172, 0x1d173, + 0x1d17a, 0x1d17b, 0xdffff, 0xe0000, 0xe0001, 0xe0fff, 0xe1000, 0x10ffff}; + +// Code points per chunk of the sweeps below: not a power of two, so chunk +// boundaries fall at varied offsets within the codecs' blocks. +constexpr size_t CODE_POINTS_PER_CHUNK = 4093; + +// Appends the scalar values in [first, last], every `step`th one, encoded by +// `encoder`, split into chunks. +void addCodePointSweep(kj::Vector>& corpus, + Encoder encoder, + char32_t first, + char32_t last, + char32_t step = 1) { + kj::Vector chunk; + size_t inChunk = 0; + for (char32_t codePoint = first; codePoint <= last; codePoint += step) { + if (codePoint >= 0xd800 && codePoint <= 0xdfff) continue; + encoder(chunk, codePoint); + if (++inChunk == CODE_POINTS_PER_CHUNK) { + corpus.add(chunk.releaseAsArray()); + inChunk = 0; + } + } + if (inChunk > 0) corpus.add(chunk.releaseAsArray()); +} + +// Appends Unicode text covering every code point that behaves distinctly: each +// supplementary edge case on its own, so a mismatch names it; every BMP code +// point; the blocks around ICU's supplementary ignorables in full; and a +// sample of the rest of the supplementary planes. +void addUnicodeInputs(kj::Vector>& corpus) { + for (Encoder encoder: {appendUtf8, appendUtf16le}) { + addCodePointSweep(corpus, encoder, 0, 0xffff); + for (auto codePoint: SUPPLEMENTARY_EDGES) { + corpus.add(encode({codePoint}, encoder)); + corpus.add(encode({U'a', codePoint, U'b'}, encoder)); + } + addCodePointSweep(corpus, encoder, 0x1bc00, 0x1bcff); + addCodePointSweep(corpus, encoder, 0x1d100, 0x1d1ff); + addCodePointSweep(corpus, encoder, 0xdff00, 0xe10ff); + addCodePointSweep(corpus, encoder, 0x10000, 0x10ffff, 251); + } +} + +kj::Array> buildCorpus() { + kj::Vector> corpus; + + // Every input of up to one byte, and every two-byte input with a sequence + // byte on either side: each UTF-8 lead byte with each kind of byte after it, + // and each UTF-16 code unit with a surrogate's or ASCII's high byte. + auto allBytes = kj::heapArray(256); + for (size_t i = 0; i < 256; ++i) allBytes[i] = static_cast(i); + addAllStrings(corpus, allBytes, 0); + addAllStrings(corpus, allBytes, 1); + for (auto first: allBytes) { + for (auto second: SEQUENCE_BYTES) { + corpus.add(kj::heapArray({first, second})); + corpus.add(kj::heapArray({second, first})); + } + } + + // Every short input of interesting bytes. + addAllStrings(corpus, SEQUENCE_BYTES, 3); + addAllStrings(corpus, SEQUENCE_BYTES, 4); + + // Longer pseudo-random inputs. + Random random; + for (size_t n = 0; n < 2000; ++n) { + corpus.add(random.input(static_cast(random.next() % 64) + 5)); + } + + // Well-formed text, in each encoding that can hold it. + constexpr char32_t TEXT[] = {U'a', U'\u00e9', U'\u2615', U'\U0001F600', U'\uFFFD', U'\uFEFF', + U'\uFFFF', U'\u200B', U'\u20AC'}; + corpus.add(encode(kj::arrayPtr(TEXT, kj::size(TEXT)), appendUtf8)); + corpus.add(encode(kj::arrayPtr(TEXT, kj::size(TEXT)), appendUtf16le)); + + addLongInputs(corpus, random); + addUnicodeInputs(corpus); + + return corpus.releaseAsArray(); +} + +KJ_TEST("Rust and C++ transcode agree") { + auto corpus = buildCorpus(); + + jsg::test::Evaluator e(v8System); + e.getIsolate().runInLockScope([&](I18nIsolate::Lock& isolateLock) { + JSG_WITHIN_CONTEXT_SCOPE(isolateLock, + isolateLock.newContext().getHandle(isolateLock), [&](jsg::Lock& js) { + for (auto from: ENCODINGS) { + for (auto to: ENCODINGS) { + useRust(false); + auto expected = KJ_MAP(source, corpus) { return transcodeOutcome(js, source, from, to); }; + useRust(true); + size_t mismatches = 0; + for (size_t i = 0; i < corpus.size(); ++i) { + auto actual = transcodeOutcome(js, corpus[i], from, to); + if (actual != expected[i] && mismatches++ < 10) { + KJ_FAIL_EXPECT("Rust and C++ transcode disagree", static_cast(from), + static_cast(to), i, describeBytes(corpus[i]), + describeMismatch(expected[i], actual)); + } + } + KJ_EXPECT(mismatches == 0, static_cast(from), static_cast(to), mismatches); + } + } + }); + }); + + util::Autogate::deinitAutogate(); +} + +} // namespace +} // namespace workerd::api::node diff --git a/src/workerd/api/node/i18n.c++ b/src/workerd/api/node/i18n.c++ index a895a7c72c0..521c276caf7 100644 --- a/src/workerd/api/node/i18n.c++ +++ b/src/workerd/api/node/i18n.c++ @@ -8,7 +8,11 @@ #include "simdutf.h" #include +#include +#include +#include +#include #include #include #include @@ -18,6 +22,40 @@ namespace workerd::api::node { +namespace rust_i18n = ::workerd::rust::i18n; + +// Maps the C++ Encoding to the Rust bridge enum, following the kj-rs +// convert.h idiom, so callers use `kj::from(value)`. It +// is `static` (rather than in an anonymous namespace) because ADL is how +// `kj::from` locates this overload, and ADL does not search an +// unnamed namespace nested inside the argument type's namespace. The switch +// has no `default:` arm so that enum drift between the two `Encoding` types is +// a compile error. +// +// `BASE64`, `BASE64URL`, and `HEX` are not transcodable (see +// `i18n::canBeTranscoded`); rejecting them here, for both `fromEncoding` and +// `toEncoding`, is a deliberate divergence from the C++ dispatch below, which +// only checks `fromEncoding` and would reach `KJ_UNREACHABLE` for a +// non-transcodable `toEncoding`. Neither is reachable from JavaScript because +// `BufferUtil::transcode` validates both encodings first. +static rust_i18n::Encoding fromImpl(rust_i18n::Encoding*, Encoding encoding) { + switch (encoding) { + case Encoding::ASCII: + return rust_i18n::Encoding::Ascii; + case Encoding::LATIN1: + return rust_i18n::Encoding::Latin1; + case Encoding::UTF8: + return rust_i18n::Encoding::Utf8; + case Encoding::UTF16LE: + return rust_i18n::Encoding::Utf16Le; + case Encoding::BASE64: + case Encoding::BASE64URL: + case Encoding::HEX: + JSG_FAIL_REQUIRE(Error, "Invalid encoding passed to transcode"); + } + KJ_UNREACHABLE; +} + namespace i18n { namespace { @@ -240,6 +278,16 @@ void Converter::setSubstituteChars(kj::StringPtr sub) { jsg::JsUint8Array transcode( jsg::Lock& js, kj::ArrayPtr source, Encoding fromEncoding, Encoding toEncoding) { + if (util::Autogate::isEnabled(util::AutogateKey::NODEJS_I18N_RUST)) { + auto rustFrom = kj::from(fromEncoding); + auto rustTo = kj::from(toEncoding); + auto maybeLocal = + rust_i18n::transcode(js.v8Isolate, source.as(), rustFrom, rustTo); + auto local = + jsg::check(::workerd::rust::jsg::maybe_local_from_ffi(kj::mv(maybeLocal))); + return jsg::JsUint8Array(local); + } + TranscodeImpl transcode_function = &TranscodeDefault; switch (fromEncoding) { case Encoding::ASCII: diff --git a/src/workerd/api/node/tests/buffer-nodejs-test.js b/src/workerd/api/node/tests/buffer-nodejs-test.js index 91b378456ba..9858cbc24eb 100644 --- a/src/workerd/api/node/tests/buffer-nodejs-test.js +++ b/src/workerd/api/node/tests/buffer-nodejs-test.js @@ -5991,6 +5991,25 @@ export const isUtf8Test = { }, }; +export const transcodeDoesNotExposeUninitializedMemoryTest = { + test(ctrl, env, ctx) { + const result = transcode(Buffer.from('ā˜•'), 'utf8', 'ascii'); + deepStrictEqual(result, Buffer.from('?')); + + const backingStore = new Uint8Array(result.buffer); + const visibleStart = result.byteOffset; + const visibleEnd = visibleStart + result.byteLength; + deepStrictEqual( + backingStore.slice(0, visibleStart), + new Uint8Array(visibleStart) + ); + deepStrictEqual( + backingStore.slice(visibleEnd), + new Uint8Array(backingStore.length - visibleEnd) + ); + }, +}; + // Adapted from test/parallel/test-icu-transcode.js export const transcodeTest = { test(ctrl, env, ctx) { @@ -6148,6 +6167,59 @@ export const transcodeFromUTF16OddByteInputTest = { }, }; +// Pins down the substitution behaviour of the ICU-backed pairs, which the +// C++ and Rust implementations must agree on. workerd's ICU data aliases +// 'us-ascii' and 'iso8859-1' to windows-1252, so for these pairs (unlike +// Node.js, and unlike 'latin1' -> 'utf16le') both encodings are windows-1252. +export const transcodeSubstitutionTest = { + test() { + const bytes = (buf) => [...buf]; + // windows-1252 decoding, and identity pairs passing high bytes through. + deepStrictEqual( + bytes(transcode(Buffer.from([0x80, 0x81, 0xe9]), 'latin1', 'utf8')), + [0xe2, 0x82, 0xac, 0xc2, 0x81, 0xc3, 0xa9] + ); + deepStrictEqual( + bytes(transcode(Buffer.from([0x61, 0x80, 0xff]), 'ascii', 'ascii')), + [0x61, 0x80, 0xff] + ); + // windows-1252 encoding: unmappable characters become '?', but + // unmappable default-ignorable code points are dropped. + deepStrictEqual( + bytes(transcode(Buffer.from('ā‚¬\u0080ā˜•a\u200bb'), 'utf8', 'latin1')), + [0x80, 0x3f, 0x3f, 0x61, 0x62] + ); + // One U+FFFD per maximal subpart of ill-formed UTF-8. + deepStrictEqual( + bytes(transcode(Buffer.from([0xf4, 0x90, 0x80, 0x80]), 'utf8', 'ascii')), + [0x3f, 0x3f, 0x3f, 0x3f] + ); + // Unpaired surrogates; a trailing odd byte, which absorbs a lead + // surrogate before it. + deepStrictEqual( + bytes( + transcode(Buffer.from([0x00, 0xd8, 0x41, 0x00]), 'utf16le', 'ascii') + ), + [0x3f, 0x41] + ); + deepStrictEqual( + bytes( + transcode( + Buffer.from([0x41, 0x00, 0x00, 0xd8, 0x42]), + 'utf16le', + 'utf16le' + ) + ), + [0x41, 0x00, 0xfd, 0xff] + ); + // The destination is sized for the worst case, and JavaScript can see it. + strictEqual( + transcode(Buffer.from([0x61]), 'latin1', 'utf8').buffer.byteLength, + 3 + ); + }, +}; + // Tests are taken from Node.js // https://github.com/nodejs/node/blob/a4f609fa/test/parallel/test-file.js export const fileTest = { diff --git a/src/workerd/util/autogate.h b/src/workerd/util/autogate.h index d5f83dca511..0b881f7bdc9 100644 --- a/src/workerd/util/autogate.h +++ b/src/workerd/util/autogate.h @@ -126,7 +126,12 @@ namespace workerd::util { V(JSRPC_READABLE_CANCEL_PROPAGATION) \ /* Enables the V8 startup-snapshot pipeline: a throwaway zygote Worker is built to produce a \ startup snapshot, and the real Worker's isolate is then created from it. */ \ - V(STARTUP_SNAPSHOT) + V(STARTUP_SNAPSHOT) \ + /* When enabled, the Node.js `i18n` transcode implementation (api::node \ + i18n::transcode) is provided by the Rust implementation (src/rust/i18n) \ + instead of the C++ implementation. The C++ implementation is retained \ + for rollback. */ \ + V(NODEJS_I18N_RUST) // clang-format on // --------------------------------------------------------------------------------------