diff --git a/build/deps/rust.MODULE.bazel b/build/deps/rust.MODULE.bazel index 94d67b6ced2..1cd274b5ed4 100644 --- a/build/deps/rust.MODULE.bazel +++ b/build/deps/rust.MODULE.bazel @@ -57,6 +57,13 @@ crate.annotation( crate = "capnpc", gen_binaries = ["capnpc-rust"], ) +crate.annotation( + # rust_icu's checked-in bindings and symbol-renaming macro are selected by this value. Keep it + # synchronized with Chromium ICU's U_ICU_VERSION_MAJOR_NUM in build/deps/gen/deps.MODULE.bazel. + build_script_env = {"BAZEL": "1"}, + crate = "rust_icu_sys", + rustc_env = {"RUST_ICU_MAJOR_VERSION_NUMBER": "78"}, +) crate.from_cargo( # TODO(cleanup): This name is a bit misleading now that we don't vendor. We can clean it up as a # followup. diff --git a/build/wd_rust_crate.bzl b/build/wd_rust_crate.bzl index 9c54e2ec80b..f0ef88d4a08 100644 --- a/build/wd_rust_crate.bzl +++ b/build/wd_rust_crate.bzl @@ -55,6 +55,7 @@ def wd_rust_crate( deps = [], link_deps = [], proc_macro_deps = [], + rustc_env = {}, data = [], test_env = {}, test_tags = [], @@ -142,6 +143,7 @@ def wd_rust_crate( srcs = srcs, deps = deps, link_deps = link_deps + ["@@//deps:rust_runtime"], + rustc_env = rustc_env, visibility = visibility, data = data, proc_macro_deps = proc_macro_deps, @@ -159,6 +161,7 @@ def wd_rust_crate( size = test_size, tags = test_tags, crate_features = crate_features, + rustc_env = rustc_env, deps = test_deps, proc_macro_deps = test_proc_macro_deps, ) diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index c9979292e20..051d6900a12 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -472,6 +472,7 @@ dependencies = [ "quote", "ruff_python_ast", "ruff_python_parser", + "rust_icu_sys", "rustversion", "scratch", "serde", @@ -957,6 +958,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.189" @@ -1158,6 +1165,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1456,6 +1469,28 @@ name = "ruff_text_size" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff?tag=0.12.1#32c54189cb45a9d0409a1140265ce6d5fcec214d" +[[package]] +name = "rust_icu_release" +version = "5.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adff2447072626b05450f000578265104ba9df89e67e781ee54add74abed97e8" +dependencies = [ + "anyhow", +] + +[[package]] +name = "rust_icu_sys" +version = "5.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bf2afe1327bd09e8d50bcf8abcea97410a10b2f10b932e994abad414038a711" +dependencies = [ + "anyhow", + "lazy_static", + "libc", + "paste", + "rust_icu_release", +] + [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index 9aa2a97789c..c50f3402aef 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -51,6 +51,9 @@ nix = "0" pico-args = "0" ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } +# Use rust_icu's checked-in bindings for the Chromium ICU version linked by workerd. ICU itself is +# supplied by Bazel, so bindgen, icu-config, and rust_icu's library-linking features stay disabled. +rust_icu_sys = { version = "5.8", default-features = false, features = ["renaming", "icu_version_in_env"] } # param_extractor depends on unbounded_depth feature serde_json = { version = "1", features = ["unbounded_depth"] } serde = { version = "1", features = ["derive"] } diff --git a/src/rust/cxx/PATCHES.md b/src/rust/cxx/PATCHES.md index 0385246b7d7..05c54583318 100644 --- a/src/rust/cxx/PATCHES.md +++ b/src/rust/cxx/PATCHES.md @@ -13,5 +13,14 @@ The in-tree fork contains these major behavioral changes: `kj::Maybe`, `kj::Date`, exceptions, promises, and Rust futures. - **`__WORKERD_CXX__`.** This preprocessor definition identifies the fork even though its crate and generated symbol names remain compatible with cxx. +- **`c_char16`.** An additional builtin type, spelled `char16_t` in C++, for UTF-16 data. `u16` + means `uint16_t`, which C++ mangles and overloads separately from `char16_t`, so a bridge that + must name `char16_t` cannot use `u16`. The Rust side is an alias for `u16`, following `c_char`, + so no cast is needed at call sites; see the doc comment on `cxx::c_char16` for the limits that + follow from being an alias. +- **Unqualified emission of C++ fundamental types.** A `Pair` whose name is a fundamental type and + whose namespace is empty is written bare rather than with a leading `::`, because those names are + keywords: `::char16_t` is ill-formed where `::uint16_t` is fine. This lets an `ExternType` alias + bind a fundamental type that has no builtin spelling in a bridge. - **Workerd build integration.** Bazel targets use workerd's in-tree Rust toolchain, crate vendor repository, Cap'n Proto/KJ dependency, formatting, and lint configuration. diff --git a/src/rust/cxx/gen/src/write.rs b/src/rust/cxx/gen/src/write.rs index 19cda863dc9..c5ae0dcaaa2 100644 --- a/src/rust/cxx/gen/src/write.rs +++ b/src/rust/cxx/gen/src/write.rs @@ -242,7 +242,9 @@ fn pick_includes_and_builtins(out: &mut OutFile, apis: &[Api]) { Some(Isize) => out.builtin.rust_isize = true, Some(CxxString) => out.include.string = true, Some(RustString) => out.builtin.rust_string = true, - Some(Bool | Char | F32 | F64) | None => {} + // bool, char, char16_t, float and double are C++ keywords; no + // header supplies them. + Some(Bool | Char | Char16 | F32 | F64) | None => {} }, Type::RustBox(_) => out.builtin.rust_box = true, Type::RustVec(_) => out.builtin.rust_vec = true, @@ -1374,6 +1376,7 @@ fn write_atom(out: &mut OutFile, atom: Atom) { match atom { Bool => write!(out, "bool"), Char => write!(out, "char"), + Char16 => write!(out, "char16_t"), U8 => write!(out, "::std::uint8_t"), U16 => write!(out, "::std::uint16_t"), U32 => write!(out, "::std::uint32_t"), diff --git a/src/rust/cxx/src/cxx.cc b/src/rust/cxx/src/cxx.cc index 7384d28992d..afc447581f2 100644 --- a/src/rust/cxx/src/cxx.cc +++ b/src/rust/cxx/src/cxx.cc @@ -437,6 +437,14 @@ static_assert(sizeof(rust::isize) == sizeof(std::intptr_t), static_assert(alignof(rust::isize) == alignof(std::intptr_t), "unsupported ssize_t alignment"); +// The cxx crate spells char16_t as `c_char16`, an alias for u16. char16_t's +// underlying type is uint_least16_t, which the standard permits to be wider +// than 16 bits; on such a target the alias would misdescribe the C++ type. +static_assert(sizeof(char16_t) == sizeof(std::uint16_t), + "unsupported char16_t size"); +static_assert(alignof(char16_t) == alignof(std::uint16_t), + "unsupported char16_t alignment"); + static_assert(std::is_trivially_copy_constructible::value, "trivial Str(const Str &)"); static_assert(std::is_trivially_copy_assignable::value, @@ -827,10 +835,14 @@ static_assert(sizeof(std::string) <= kMaxExpectedWordsInString * sizeof(void *), FOR_EACH_TRIVIAL_STD_VECTOR(MACRO) \ MACRO(string, std::string) +// char16_t needs no _if_unique guard: [basic.fundamental] makes it a distinct +// type from every other fundamental type, so rust::Vec can never +// collide with another specialization. #define FOR_EACH_RUST_VEC(MACRO) \ FOR_EACH_NUMERIC(MACRO) \ MACRO(bool, bool) \ MACRO(char, rust::detail::char_if_unique) \ + MACRO(char16_t, char16_t) \ MACRO(usize, rust::detail::usize_if_unique) \ MACRO(isize, rust::detail::isize_if_unique) \ MACRO(string, rust::String) \ diff --git a/src/rust/cxx/src/lib.rs b/src/rust/cxx/src/lib.rs index 993f5f90a49..fb5c62d9ac6 100644 --- a/src/rust/cxx/src/lib.rs +++ b/src/rust/cxx/src/lib.rs @@ -467,6 +467,26 @@ pub type String = CxxString; /// import and use `CxxVector`. pub type Vector = CxxVector; +/// Rust spelling of C++'s `char16_t`, for UTF-16 data crossing the bridge. +/// +/// `char16_t` and `uint16_t` are layout-identical but distinct C++ types, so +/// C++ mangles them differently and overloads on them separately. `u16` in a +/// bridge means `uint16_t`; use `c_char16` where the signature must name +/// `char16_t`, which is the usual case for UTF-16 buffers. +/// +/// This is a plain alias rather than a newtype, following `c_char`, so a +/// `&[c_char16]` argument accepts a `&[u16]` from `str::encode_utf16` or +/// `encoding_rs` with no cast. The alias means `c_char16` and `u16` are one +/// Rust type; only their C++ spelling differs. The two therefore cannot both +/// be used as the element type of a `CxxVector` in the same program, and +/// `CxxVector` is rejected for the same reason `CxxVector` +/// is. +/// +/// `char16_t` is guaranteed by the C++ standard to be a distinct type whose +/// underlying type is `uint_least16_t`. `cxx.cc` asserts that it really is 16 +/// bits wide on the target, which is what makes this alias sound. +pub type c_char16 = u16; + // Not public API. #[doc(hidden)] pub mod private { diff --git a/src/rust/cxx/src/symbols/rust_vec.rs b/src/rust/cxx/src/symbols/rust_vec.rs index 23540f0796c..52b3981ea34 100644 --- a/src/rust/cxx/src/symbols/rust_vec.rs +++ b/src/rust/cxx/src/symbols/rust_vec.rs @@ -3,6 +3,7 @@ use core::ffi::c_char; use core::mem; use core::ptr; +use crate::c_char16; use crate::rust_string::RustString; use crate::rust_vec::RustVec; @@ -78,5 +79,9 @@ rust_vec_shims_for_primitive!(f32); rust_vec_shims_for_primitive!(f64); rust_vec_shims!("char", c_char); +// c_char16 is an alias for u16, so these duplicate the u16 shims above under a +// second set of export names. The C++ side reaches them through +// rust::Vec, which mangles to the char16_t segment. +rust_vec_shims!("char16_t", c_char16); rust_vec_shims!("string", RustString); rust_vec_shims!("str", &str); diff --git a/src/rust/cxx/syntax/atom.rs b/src/rust/cxx/syntax/atom.rs index 135f7ba4d56..2982904deda 100644 --- a/src/rust/cxx/syntax/atom.rs +++ b/src/rust/cxx/syntax/atom.rs @@ -9,6 +9,10 @@ use crate::Type; pub enum Atom { Bool, Char, // C char, not Rust char + // C++ char16_t. Layout-identical to u16 but a distinct C++ type, so it is a + // distinct atom: it exists to make overload resolution and name mangling on + // the C++ side pick char16_t rather than uint16_t. + Char16, U8, U16, U32, @@ -35,6 +39,7 @@ impl Atom { match s { "bool" => Some(Bool), "c_char" => Some(Char), + "c_char16" => Some(Char16), "u8" => Some(U8), "u16" => Some(U16), "u32" => Some(U32), @@ -66,6 +71,7 @@ impl AsRef for Atom { match self { Bool => "bool", Char => "c_char", + Char16 => "c_char16", U8 => "u8", U16 => "u16", U32 => "u32", diff --git a/src/rust/cxx/syntax/check.rs b/src/rust/cxx/syntax/check.rs index d15a7a09a24..aa4a5854d9b 100644 --- a/src/rust/cxx/syntax/check.rs +++ b/src/rust/cxx/syntax/check.rs @@ -163,8 +163,8 @@ fn check_type_rust_vec(cx: &mut Check, ty: &Ty1) { match Atom::from(&ident.rust) { None | Some( - Bool | Char | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 - | F64 | RustString, + Bool | Char | Char16 | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 + | Isize | F32 | F64 | RustString, ) => return, Some(CxxString) => {} } @@ -266,7 +266,7 @@ fn check_type_shared_ptr(cx: &mut Check, ptr: &Ty1) { Bool | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64 | CxxString, ) => return, - Some(Char | RustString) => {} + Some(Char | Char16 | RustString) => {} } } else if let Type::CxxVector(_) = &ptr.inner { cx.error(ptr, "std::shared_ptr is not supported yet"); @@ -289,7 +289,7 @@ fn check_type_weak_ptr(cx: &mut Check, ptr: &Ty1) { Bool | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64 | CxxString, ) => return, - Some(Char | RustString) => {} + Some(Char | Char16 | RustString) => {} } } else if let Type::CxxVector(_) = &ptr.inner { cx.error(ptr, "std::weak_ptr is not supported yet"); @@ -363,7 +363,7 @@ fn check_type_cxx_vector(cx: &mut Check, ptr: &Ty1) { | Some( U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 | Isize | F32 | F64 | CxxString, ) => return, - Some(Char) => { /* todo */ } + Some(Char | Char16) => { /* todo */ } Some(Bool | RustString) => {} } } @@ -900,6 +900,8 @@ fn describe(cx: &mut Check, ty: &Type) -> String { "C++ string".to_owned() } else if Atom::from(&ident.rust) == Some(Char) { "C char".to_owned() + } else if Atom::from(&ident.rust) == Some(Char16) { + "C++ char16_t".to_owned() } else { ident.rust.to_string() } diff --git a/src/rust/cxx/syntax/names.rs b/src/rust/cxx/syntax/names.rs index c62d384021f..acf8679feb0 100644 --- a/src/rust/cxx/syntax/names.rs +++ b/src/rust/cxx/syntax/names.rs @@ -32,6 +32,15 @@ impl Pair { } pub fn to_fully_qualified(&self) -> String { + // A fundamental type's name is a keyword, not something declared in a + // scope, so it is spelled bare: `::char16_t` is ill-formed where + // `::uint16_t` is fine. Reaching this with a non-empty namespace means + // the name was not really a fundamental type, so leave it qualified + // and let C++ report it. + if self.namespace.is_empty() && self.cxx.is_fundamental() { + return self.cxx.to_string(); + } + let mut fully_qualified = String::new(); for segment in &self.namespace { fully_qualified += "::"; @@ -72,6 +81,35 @@ impl ForeignName { Err(err) => Err(Error::new(span, err)), } } + + /// True if this names a C++ fundamental type. + /// + /// These are the only C++ type names that are keywords, and so the only + /// ones that cannot be written with a `::` qualification. Fixed-width + /// names like `uint16_t` are ordinary typedefs, not keywords, and are + /// deliberately absent. + /// + /// Multi-token spellings such as `unsigned int` are absent because + /// [`ForeignName::parse`] cannot represent a name containing whitespace. + pub fn is_fundamental(&self) -> bool { + matches!( + self.text.as_str(), + "bool" + | "char" + | "char8_t" + | "char16_t" + | "char32_t" + | "double" + | "float" + | "int" + | "long" + | "short" + | "signed" + | "unsigned" + | "void" + | "wchar_t" + ) + } } impl Display for ForeignName { diff --git a/src/rust/cxx/syntax/namespace.rs b/src/rust/cxx/syntax/namespace.rs index da8a335da68..98c1d36be69 100644 --- a/src/rust/cxx/syntax/namespace.rs +++ b/src/rust/cxx/syntax/namespace.rs @@ -33,6 +33,11 @@ impl Namespace { self.segments.iter() } + /// True for the global namespace. + pub fn is_empty(&self) -> bool { + self.segments.is_empty() + } + pub fn parse_bridge_attr_namespace(input: ParseStream) -> Result { if input.is_empty() { return Ok(Self::ROOT); diff --git a/src/rust/cxx/syntax/pod.rs b/src/rust/cxx/syntax/pod.rs index e8a50730fce..1e5598c1905 100644 --- a/src/rust/cxx/syntax/pod.rs +++ b/src/rust/cxx/syntax/pod.rs @@ -12,8 +12,8 @@ impl Types<'_> { let ident = &ident.rust; if let Some(atom) = Atom::from(ident) { match atom { - Bool | Char | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 | I64 - | Isize | F32 | F64 => true, + Bool | Char | Char16 | U8 | U16 | U32 | U64 | Usize | I8 | I16 | I32 + | I64 | Isize | F32 | F64 => true, CxxString | RustString => false, } } else if let Some(strct) = self.structs.get(ident) { diff --git a/src/rust/cxx/syntax/tokens.rs b/src/rust/cxx/syntax/tokens.rs index eabb34f5576..a8d5644b683 100644 --- a/src/rust/cxx/syntax/tokens.rs +++ b/src/rust/cxx/syntax/tokens.rs @@ -36,6 +36,11 @@ impl ToTokens for Type { if ident.rust == Char { let span = ident.rust.span(); tokens.extend(quote_spanned!(span=> ::cxx::core::ffi::)); + } else if ident.rust == Char16 { + // Unlike c_char, there is no core::ffi equivalent of + // char16_t; the cxx crate defines c_char16 itself. + let span = ident.rust.span(); + tokens.extend(quote_spanned!(span=> ::cxx::)); } else if ident.rust == CxxString { let span = ident.rust.span(); tokens.extend(quote_spanned!(span=> ::cxx::)); diff --git a/src/rust/cxx/tests/cxx_gen.rs b/src/rust/cxx/tests/cxx_gen.rs index e9f68414c12..3f6122cbe11 100644 --- a/src/rust/cxx/tests/cxx_gen.rs +++ b/src/rust/cxx/tests/cxx_gen.rs @@ -88,6 +88,33 @@ const BRIDGE4: &str = r#" } "#; +const BRIDGE5: &str = r#" + #[cxx::bridge] + mod ffi { + unsafe extern "C++" { + fn utf16_length(s: &[c_char16]) -> usize; + fn widen(input: &[u8], output: &mut [c_char16]) -> usize; + unsafe fn first(s: *const c_char16) -> c_char16; + } + + extern "Rust" { + fn encode(s: &str) -> Vec; + } + } +"#; + +const BRIDGE6: &str = r#" + #[cxx::bridge] + mod ffi { + unsafe extern "C++" { + #[cxx_name = "char32_t"] + type Char32 = crate::Char32; + + fn count(s: &[Char32]) -> usize; + } + } +"#; + #[test] fn test_extern_c_function() { let opt = cxx_gen::Opt::default(); @@ -177,3 +204,30 @@ fn test_kj_arc_in_shared_struct() { let expected = "::rust::ManuallyDrop<::Holder> holder$(::std::move(holder));"; assert!(implementation.contains(expected)); } + +#[test] +fn test_c_char16_maps_to_cxx_char16_t() { + let opt = cxx_gen::Opt::default(); + let source = BRIDGE5.parse().unwrap(); + let generated = generate_header_and_cc(source, &opt).unwrap(); + let implementation = str::from_utf8(&generated.implementation).unwrap(); + assert!(implementation.contains("::rust::Slice")); + assert!(implementation.contains("::rust::Slice")); + assert!(implementation.contains("char16_t const *")); + assert!(implementation.contains("::rust::Vec")); + // char16_t must not decay to uint16_t: keeping the two distinct is the + // whole reason c_char16 is a separate atom from u16. + assert!(!implementation.contains("uint16_t")); +} + +#[test] +fn test_fundamental_type_name_is_unqualified() { + let opt = cxx_gen::Opt::default(); + let source = BRIDGE6.parse().unwrap(); + let generated = generate_header_and_cc(source, &opt).unwrap(); + let implementation = str::from_utf8(&generated.implementation).unwrap(); + assert!(implementation.contains("::rust::Slice")); + // `::char32_t` is ill-formed; a fundamental type's name is a keyword and + // cannot be qualified. + assert!(!implementation.contains("::char32_t")); +} diff --git a/src/rust/cxx/tests/ffi/lib.rs b/src/rust/cxx/tests/ffi/lib.rs index 03af290cc69..f10798c8ef7 100644 --- a/src/rust/cxx/tests/ffi/lib.rs +++ b/src/rust/cxx/tests/ffi/lib.rs @@ -29,6 +29,7 @@ use cxx::KjError; use cxx::KjExceptionType; use cxx::SharedPtr; use cxx::UniquePtr; +use cxx::c_char16; use cxx::type_id; // The bridge parser accepts the unqualified smart-pointer name, while expansion // fully qualifies the emitted Rust field type. @@ -153,6 +154,8 @@ pub mod ffi { unsafe fn c_return_mut<'a>(shared: &'a mut Shared) -> &'a mut usize; unsafe fn c_return_str<'a>(shared: &'a Shared) -> &'a str; unsafe fn c_return_slice_char<'a>(shared: &'a Shared) -> &'a [c_char]; + unsafe fn c_return_slice_char16<'a>(shared: &'a Shared) -> &'a [c_char16]; + fn c_return_rust_vec_char16() -> Vec; unsafe fn c_return_mutsliceu8<'a>(slice: &'a mut [u8]) -> &'a mut [u8]; unsafe fn c_return_ref<'a>(shared: &'a Shared) -> &'a usize; fn c_return_rust_string() -> String; @@ -245,6 +248,8 @@ pub mod ffi { fn c_take_ref_c(c: &C); fn c_take_str(s: &str); fn c_take_slice_char(s: &[c_char]); + fn c_take_slice_char16(s: &[c_char16]); + fn c_take_rust_vec_char16(v: Vec); fn c_take_slice_shared(s: &[Shared]); fn c_take_slice_shared_sort(s: &mut [Shared]); fn c_take_slice_r(s: &[R]); @@ -430,6 +435,8 @@ pub mod ffi { fn r_take_ref_c(c: &C); fn r_take_str(s: &str); fn r_take_slice_char(s: &[c_char]); + fn r_take_slice_char16(s: &[c_char16]); + fn r_return_rust_vec_char16() -> Vec; fn r_take_rust_string(s: String); fn r_take_unique_ptr_string(s: UniquePtr); fn r_take_ref_vector(v: &CxxVector); @@ -758,6 +765,16 @@ fn r_take_slice_char(s: &[c_char]) { assert_eq!(std::str::from_utf8(s), Ok("2020\0")); } +// c_char16 is an alias for u16, so a &[c_char16] is a &[u16] and UTF-16 from +// the standard library needs no conversion to cross the bridge. +fn r_take_slice_char16(s: &[c_char16]) { + assert_eq!(String::from_utf16_lossy(s), "2020"); +} + +fn r_return_rust_vec_char16() -> Vec { + "2020".encode_utf16().collect() +} + fn r_take_unique_ptr_string(s: UniquePtr) { assert_eq!( s.as_ref().and_then(|value| value.to_str().ok()), diff --git a/src/rust/cxx/tests/ffi/tests.cc b/src/rust/cxx/tests/ffi/tests.cc index ea6de70cca6..950afb5e2b2 100644 --- a/src/rust/cxx/tests/ffi/tests.cc +++ b/src/rust/cxx/tests/ffi/tests.cc @@ -24,6 +24,7 @@ extern "C" bool cxx_test_suite_r_is_correct(const tests::R *) noexcept; namespace tests { static constexpr char SLICE_DATA[] = "2020"; +static constexpr char16_t SLICE_DATA16[] = u"2020"; static_assert(sizeof(SharedWithKjOwn) == sizeof(kj::Own)); static_assert(alignof(SharedWithKjOwn) == alignof(kj::Own)); @@ -153,6 +154,21 @@ rust::Slice c_return_slice_char(const Shared &shared) { return rust::Slice(SLICE_DATA, sizeof(SLICE_DATA)); } +rust::Slice c_return_slice_char16(const Shared &shared) { + (void)shared; + // Excludes the trailing NUL, unlike c_return_slice_char, so the Rust side can + // compare against "2020" directly. + return rust::Slice(SLICE_DATA16, 4); +} + +rust::Vec c_return_rust_vec_char16() { + rust::Vec vec; + for (char16_t c : rust::Slice(SLICE_DATA16, 4)) { + vec.push_back(c); + } + return vec; +} + rust::Slice c_return_mutsliceu8(rust::Slice slice) { return slice; } @@ -517,6 +533,18 @@ void c_take_slice_char(rust::Slice s) { } } +void c_take_slice_char16(rust::Slice s) { + if (std::u16string(s.data(), s.size()) == u"2020") { + cxx_test_suite_set_correct(); + } +} + +void c_take_rust_vec_char16(rust::Vec v) { + if (std::u16string(v.data(), v.size()) == u"2020") { + cxx_test_suite_set_correct(); + } +} + void c_take_slice_shared(rust::Slice s) { if (s.size() == 2 && s.data()->z == 2020 && s[1].z == 2021 && s.at(1).z == 2021 && s.front().z == 2020 && s.back().z == 2021) { @@ -1090,6 +1118,8 @@ extern "C" const char *cxx_run_test() noexcept { r_take_ref_c(C{2020}); r_take_str(rust::Str("2020")); r_take_slice_char(rust::Slice(SLICE_DATA, sizeof(SLICE_DATA))); + r_take_slice_char16(rust::Slice(SLICE_DATA16, 4)); + ASSERT(std::u16string(r_return_rust_vec_char16().data(), 4) == u"2020"); r_take_rust_string(rust::String("2020")); r_take_unique_ptr_string( std::unique_ptr(new std::string("2020"))); diff --git a/src/rust/cxx/tests/ffi/tests.h b/src/rust/cxx/tests/ffi/tests.h index 556407d44ba..b04da061332 100644 --- a/src/rust/cxx/tests/ffi/tests.h +++ b/src/rust/cxx/tests/ffi/tests.h @@ -151,6 +151,8 @@ const size_t &c_return_nested_ns_ref(const ::A::B::ABShared &shared); size_t &c_return_mut(Shared &shared); rust::Str c_return_str(const Shared &shared); rust::Slice c_return_slice_char(const Shared &shared); +rust::Slice c_return_slice_char16(const Shared &shared); +rust::Vec c_return_rust_vec_char16(); rust::Slice c_return_mutsliceu8(rust::Slice slice); rust::String c_return_rust_string(); rust::String c_return_rust_string_lossy(); @@ -230,6 +232,8 @@ void c_take_ref_c(const C &c); void c_take_ref_ns_c(const ::H::H &h); void c_take_str(rust::Str s); void c_take_slice_char(rust::Slice s); +void c_take_slice_char16(rust::Slice s); +void c_take_rust_vec_char16(rust::Vec v); void c_take_slice_shared(rust::Slice s); void c_take_slice_shared_sort(rust::Slice s); void c_take_slice_r(rust::Slice s); diff --git a/src/rust/cxx/tests/test.rs b/src/rust/cxx/tests/test.rs index cb5b6089226..db7660856e1 100644 --- a/src/rust/cxx/tests/test.rs +++ b/src/rust/cxx/tests/test.rs @@ -61,7 +61,17 @@ fn test_c_return() { b"2020\0", cast::c_char_to_unsigned(ffi::c_return_slice_char(&shared)), ); + // c_char16 is an alias for u16, so a rust::Slice from + // C++ arrives as a plain &[u16] with no cast. + assert_eq!( + String::from_utf16(ffi::c_return_slice_char16(&shared)).unwrap(), + "2020", + ); } + assert_eq!( + String::from_utf16(&ffi::c_return_rust_vec_char16()).unwrap(), + "2020", + ); assert_eq!("2020", ffi::c_return_rust_string()); assert_eq!("Hello \u{fffd}World", ffi::c_return_rust_string_lossy()); assert_eq!("2020", ffi::c_return_unique_ptr_string().to_str().unwrap()); @@ -350,6 +360,9 @@ fn test_c_take() { check!(cxx_test_suite::module::ffi::c_take_unique_ptr(unique_ptr)); check!(ffi::c_take_str("2020")); check!(ffi::c_take_slice_char(cast::unsigned_to_c_char(b"2020"))); + let utf16: Vec = "2020".encode_utf16().collect(); + check!(ffi::c_take_slice_char16(&utf16)); + check!(ffi::c_take_rust_vec_char16(utf16)); check!(ffi::c_take_slice_shared(&[ ffi::Shared { z: 2020 }, ffi::Shared { z: 2021 }, diff --git a/src/rust/i18n/BUILD.bazel b/src/rust/i18n/BUILD.bazel new file mode 100644 index 00000000000..34fa251a2a4 --- /dev/null +++ b/src/rust/i18n/BUILD.bazel @@ -0,0 +1,24 @@ +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +wd_rust_crate( + name = "i18n", + cxx_bridge_deps = [ + "//src/rust/jsg", + "@simdutf", + "@workerd-v8//:v8", + # Also depends on ICU for bindings, which are extracted from the v8 dep above + ], + cxx_bridge_src = "lib.rs", + link_deps = [ + "@simdutf", + "@workerd-v8//:v8", + ], + rustc_env = {"RUST_ICU_MAJOR_VERSION_NUMBER": "78"}, + test_deps = ["//src/rust/jsg-test"], + visibility = ["//visibility:public"], + deps = [ + "//src/rust/jsg", + "@crates_vendor//:rust_icu_sys", + "@crates_vendor//:thiserror", + ], +) diff --git a/src/rust/i18n/codecs.rs b/src/rust/i18n/codecs.rs new file mode 100644 index 00000000000..0ce374747bf --- /dev/null +++ b/src/rust/i18n/codecs.rs @@ -0,0 +1,263 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Safe wrappers around the ICU primitives exposed by `rust_icu_sys` and the +//! simdutf primitives declared in [`crate::ffi`]. Both are raw entry points +//! taking bare pointers. +//! +//! Everything unsafe about calling them lives here: deriving pointers and +//! lengths from slices, owning the `UConverter`, and turning their C-ish +//! reporting conventions (an out-parameter `UErrorCode`, `-1`, `0`) into +//! `Option` and `Result`. The transcoding *logic* -- dispatch, sizing, +//! substitute-character setup, truncation -- lives in [`crate::dispatch`]. + +use std::ffi::CStr; +use std::ffi::c_char; + +use rust_icu_sys as sys; +use rust_icu_sys::versioned_function; + +use crate::error::TranscodeError; +use crate::ffi; + +/// ICU treats positive codes as failures and negative codes as warnings, +/// matching the `U_FAILURE` macro. +fn is_failure(error: sys::UErrorCode) -> bool { + error > sys::UErrorCode::U_ZERO_ERROR +} + +/// Returns the ICU converter name for a transcodable encoding, matching +/// `getEncodingName()` in `i18n.c++`. +/// +/// The bridge `Encoding` enum is a `cxx` shared enum, which is a `u8` newtype +/// rather than a real Rust enum, so a value outside the four declared variants +/// is representable. It can only arise if the C++ and Rust halves of the +/// bridge disagree, and is reported as an error rather than a panic because a +/// panic crossing the bridge aborts the process. +fn icu_name(encoding: ffi::Encoding) -> Result<&'static CStr, TranscodeError> { + match encoding { + ffi::Encoding::Ascii => Ok(c"us-ascii"), + ffi::Encoding::Latin1 => Ok(c"iso8859-1"), + ffi::Encoding::Utf16Le => Ok(c"utf16le"), + ffi::Encoding::Utf8 => Ok(c"utf-8"), + _ => Err(TranscodeError::InvalidEncoding), + } +} + +/// An open ICU converter for one of the four transcodable encodings. +/// +/// Owns its `UConverter` and closes it on drop, including while unwinding. +/// Holding a raw pointer makes the type neither `Send` nor `Sync`, which is +/// what we want: ICU converters carry conversion state and are not safe to +/// share between threads. +pub struct Converter(*mut sys::UConverter); + +impl Converter { + /// Opens an ICU converter for `encoding`. + pub fn open(encoding: ffi::Encoding) -> Result { + let mut err = sys::UErrorCode::U_ZERO_ERROR; + // SAFETY: `icu_name` returns a NUL-terminated static string, and `err` + // is a live local for the duration of the call. + let cnv = + unsafe { versioned_function!(ucnv_open)(icu_name(encoding)?.as_ptr(), &raw mut err) }; + if is_failure(err) || cnv.is_null() { + return Err(TranscodeError::ConverterOpenFailed); + } + Ok(Self(cnv)) + } + + /// Returns the largest number of bytes a single character occupies in this + /// converter's encoding. + pub fn max_char_size(&self) -> usize { + // SAFETY: `self.0` is non-null for as long as `self` is alive. + let size = unsafe { versioned_function!(ucnv_getMaxCharSize)(self.0) }; + // ICU returns a positive byte count; the cast cannot lose information. + size.unsigned_abs().into() + } + + /// Returns the smallest number of bytes a single character occupies in this + /// converter's encoding. + pub fn min_char_size(&self) -> usize { + // SAFETY: `self.0` is non-null for as long as `self` is alive. + let size = unsafe { versioned_function!(ucnv_getMinCharSize)(self.0) }; + size.unsigned_abs().into() + } + + /// Sets the converter's substitute character sequence, used in place of + /// unmappable characters during conversion. + /// + /// Without this ICU substitutes its own default, which for ASCII is + /// U+001A rather than the `?` the C++ path produces. + pub fn set_subst_chars(&self, substitute: &str) -> Result<(), TranscodeError> { + if substitute.is_empty() { + return Ok(()); + } + // ICU takes the length as an `int8_t` and reads a negative length as + // "NUL-terminated", which `substitute` is not. Its own limit on + // substitute sequences is far lower still. + let length = + i8::try_from(substitute.len()).map_err(|_| TranscodeError::SetSubstituteCharsFailed)?; + + let mut err = sys::UErrorCode::U_ZERO_ERROR; + // SAFETY: `self.0` is non-null, and `substitute` outlives the call and + // is at least `length` bytes long. ICU takes the sequence as bytes and + // does not require NUL termination when given an explicit length. + unsafe { + versioned_function!(ucnv_setSubstChars)( + self.0, + substitute.as_ptr().cast(), + length, + &raw mut err, + ); + } + if is_failure(err) { + return Err(TranscodeError::SetSubstituteCharsFailed); + } + Ok(()) + } +} + +impl Drop for Converter { + fn drop(&mut self) { + // SAFETY: `self.0` was returned non-null by `ucnv_open` and is closed + // exactly once, here. + unsafe { versioned_function!(ucnv_close)(self.0) } + } +} + +/// Converts `source` from `from`'s encoding to `to`'s encoding via ICU's +/// `ucnv_convertEx`, mirroring `TranscodeDefault` in `i18n.c++`. Returns the +/// number of bytes written to `target`, or `None` if ICU reports failure. +pub fn convert_ex( + to: &Converter, + from: &Converter, + source: &[u8], + target: &mut [u8], +) -> Option { + let target_start: *mut c_char = target.as_mut_ptr().cast(); + let mut target_cursor = target_start; + let mut source_cursor: *const c_char = source.as_ptr().cast(); + let mut err = sys::UErrorCode::U_ZERO_ERROR; + + // SAFETY: both cursors start at the base of a live slice and are bounded + // by a limit one past that slice's end, which is what ICU advances them + // against. Passing a null pivot asks ICU to use an internal one. + unsafe { + versioned_function!(ucnv_convertEx)( + to.0, + from.0, + &raw mut target_cursor, + target_start.add(target.len()), + &raw mut source_cursor, + source_cursor.add(source.len()), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null(), + 1, // reset + 1, // flush + &raw mut err, + ); + } + if is_failure(err) { + return None; + } + // SAFETY: ICU advanced `target_cursor` within `target`, so both pointers + // are into the same allocation. + let written = unsafe { target_cursor.offset_from(target_start) }; + usize::try_from(written).ok() +} + +/// Converts UTF-16LE `source` (as raw bytes) to `to`'s encoding via ICU's +/// `ucnv_fromUChars`, mirroring `TranscodeFromUTF16` in `i18n.c++`. Returns +/// the number of bytes written to `target`, or `None` if ICU reports failure. +pub fn from_uchars(to: &Converter, source: &[u8], target: &mut [u8]) -> Option { + let src_length = i32::try_from(source.len() / size_of::()).ok()?; + let dest_capacity = i32::try_from(target.len()).ok()?; + let mut err = sys::UErrorCode::U_ZERO_ERROR; + + // SAFETY: the pointers and lengths describe the two live slices. `source` + // need not be `u16`-aligned -- it is caller-supplied buffer contents, which + // a Uint8Array can expose at an odd byteOffset -- and is reinterpreted as + // `UChar*` exactly as `i18n.c++` does with the same bytes. Casting a raw + // pointer is well-defined in Rust regardless of alignment; no reference to + // the misaligned data is ever formed on this side. + let len = unsafe { + versioned_function!(ucnv_fromUChars)( + to.0, + target.as_mut_ptr().cast(), + dest_capacity, + source.as_ptr().cast(), + src_length, + &raw mut err, + ) + }; + if is_failure(err) { + return None; + } + usize::try_from(len).ok() +} + +/// Widens Latin-1 `source` into UTF-16 (written to `target` as raw bytes), +/// mirroring `simdutf::convert_latin1_to_utf16`. Returns the number of +/// `char16_t` units written. +pub fn convert_latin1_to_utf16(source: &[u8], target: &mut [u8]) -> usize { + // SAFETY: `target` holds at least `source.len()` UTF-16 units, which + // `dispatch` guarantees by sizing it at two bytes per source byte. See + // `from_uchars` on the alignment of these casts. + unsafe { + ffi::convert_latin1_to_utf16( + source.as_ptr().cast(), + source.len(), + target.as_mut_ptr().cast(), + ) + } +} + +/// Estimates the UTF-16 length (in `char16_t` units) of UTF-8 `source`, +/// mirroring `simdutf::utf16_length_from_utf8`. +pub fn utf16_length_from_utf8(source: &[u8]) -> usize { + // SAFETY: pointer and length describe a live slice. + unsafe { ffi::utf16_length_from_utf8(source.as_ptr().cast(), source.len()) } +} + +/// Converts UTF-8 `source` to UTF-16LE (written to `target` as raw bytes), +/// mirroring `simdutf::convert_utf8_to_utf16le`. Returns the number of +/// `char16_t` units written, or `0` on invalid UTF-8. +pub fn convert_utf8_to_utf16le(source: &[u8], target: &mut [u8]) -> usize { + // SAFETY: `target` is sized by `dispatch` at two bytes per unit that + // `utf16_length_from_utf8` reported for this same `source`, which is the + // most this call can write. + unsafe { + ffi::convert_utf8_to_utf16le( + source.as_ptr().cast(), + source.len(), + target.as_mut_ptr().cast(), + ) + } +} + +/// Estimates the UTF-8 length (in bytes) of UTF-16LE `source` (as raw bytes), +/// mirroring `simdutf::utf8_length_from_utf16le`. +pub fn utf8_length_from_utf16le(source: &[u8]) -> usize { + // SAFETY: pointer and length describe a live slice. + unsafe { + ffi::utf8_length_from_utf16le(source.as_ptr().cast(), source.len() / size_of::()) + } +} + +/// Converts UTF-16LE `source` (as raw bytes) to UTF-8, mirroring +/// `simdutf::convert_utf16le_to_utf8`. Returns the number of bytes written. +pub fn convert_utf16le_to_utf8(source: &[u8], target: &mut [u8]) -> usize { + // SAFETY: `target` is sized by `dispatch` at the byte count + // `utf8_length_from_utf16le` reported for this same `source`, which is the + // most this call can write. + unsafe { + ffi::convert_utf16le_to_utf8( + source.as_ptr().cast(), + source.len() / size_of::(), + target.as_mut_ptr().cast(), + ) + } +} diff --git a/src/rust/i18n/dispatch.rs b/src/rust/i18n/dispatch.rs new file mode 100644 index 00000000000..6298150ee55 --- /dev/null +++ b/src/rust/i18n/dispatch.rs @@ -0,0 +1,428 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Encoding-pair dispatch and the conversion logic behind each pair, ported +//! from `i18n.c++`'s `TranscodeDefault` / `TranscodeLatin1ToUTF16` / +//! `TranscodeFromUTF16` / `TranscodeUTF16FromUTF8` / `TranscodeUTF8FromUTF16` +//! and the `switch` in `transcode()` that picks between them. +//! +//! A transcode runs in two steps. [`Transcoder::new`] validates the source, +//! picks the conversion, and computes [`Transcoder::dest_len`] -- the exact +//! size of the destination buffer that conversion needs. +//! [`Transcoder::transcode_into`] then fills a buffer of that size and reports +//! how many bytes it actually wrote, which is often fewer because the +//! destination is sized for the worst case. +//! +//! All sizing, validation, substitute-character setup, and length checking +//! happens here; [`crate::codecs`] only forwards to the underlying ICU and +//! simdutf calls. + +use crate::codecs; +use crate::codecs::Converter; +use crate::error::TranscodeError; +use crate::ffi::Encoding; + +/// An isolate has a 128MB memory limit, and thus so does any single +/// destination buffer. Mirrors `ISOLATE_LIMIT` in `i18n.c++`. +const ISOLATE_LIMIT: usize = 134_217_728; + +/// A validated, sized transcode, ready to run. +/// +/// Borrows its source for the whole of its life, so [`Transcoder::dest_len`] +/// cannot go stale: the bytes it was computed from are the same bytes +/// [`Transcoder::transcode_into`] reads. This matters because three of the +/// five conversions bottom out in simdutf functions that take no output +/// length and size their writes purely from the source. +pub struct Transcoder<'a> { + source: &'a [u8], + conversion: Conversion, + dest_len: usize, +} + +/// The conversion [`Transcoder::transcode_into`] will perform, along with any +/// ICU converters [`Transcoder::new`] had to open to size the destination. +enum Conversion { + /// ICU `ucnv_convertEx` between two converters, mirroring + /// `TranscodeDefault`. Handles every pair the simdutf conversions below do + /// not, including all four identity pairs. + ConvertEx { to: Converter, from: Converter }, + /// simdutf Latin-1 to UTF-16, mirroring `TranscodeLatin1ToUTF16`. + Latin1ToUtf16, + /// ICU `ucnv_fromUChars` from UTF-16LE, mirroring `TranscodeFromUTF16`. + FromUtf16 { to: Converter }, + /// simdutf UTF-8 to UTF-16LE, mirroring `TranscodeUTF16FromUTF8`. + Utf16FromUtf8, + /// simdutf UTF-16LE to UTF-8, mirroring `TranscodeUTF8FromUTF16`. + Utf8FromUtf16, +} + +impl<'a> Transcoder<'a> { + /// Prepares a transcode of `source` from `from` to `to`, matching the + /// dispatch table built by `i18n::transcode()` in `i18n.c++`. + /// + /// Returns an error if `source` is malformed for `from`, or if the + /// destination the conversion would need exceeds [`ISOLATE_LIMIT`]. + pub fn new(source: &'a [u8], from: Encoding, to: Encoding) -> Result { + match (from, to) { + (Encoding::Ascii | Encoding::Latin1, Encoding::Utf16Le) => { + Self::latin1_to_utf16(source) + } + (Encoding::Utf8, Encoding::Utf16Le) => Self::utf16_from_utf8(source), + (Encoding::Utf16Le, Encoding::Utf8) => Self::utf8_from_utf16(source), + (Encoding::Utf16Le, Encoding::Ascii | Encoding::Latin1) => Self::from_utf16(source, to), + // Identity pairs, UTF16LE -> UTF16LE, and anything else the + // simdutf conversions above do not cover. + _ => Self::convert_ex(source, from, to), + } + } + + /// The exact size, in bytes, of the destination buffer + /// [`Transcoder::transcode_into`] requires. + pub fn dest_len(&self) -> usize { + self.dest_len + } + + /// Transcodes into `dest`, returning the number of bytes written, which + /// may be fewer than `dest.len()`. + /// + /// `dest` must be exactly [`Transcoder::dest_len`] bytes long. + pub fn transcode_into(&self, dest: &mut [u8]) -> Result { + // Not merely a sanity check: the simdutf conversions below take no + // output length, so a destination shorter than the size computed for + // this source would overflow it. + if dest.len() != self.dest_len { + return Err(TranscodeError::DestinationSizeMismatch); + } + // A zero-length destination means the conversion has nothing to write: + // either the source was empty, or the estimated output length was zero. + if dest.is_empty() { + return Ok(0); + } + + let source = self.source; + match &self.conversion { + Conversion::ConvertEx { to, from } => { + codecs::convert_ex(to, from, source, dest).ok_or(TranscodeError::UnableToTranscode) + } + Conversion::Latin1ToUtf16 => { + let units = codecs::convert_latin1_to_utf16(source, dest); + // simdutf returns 0 for invalid input. + if units == 0 { + return Err(TranscodeError::UnableToTranscode); + } + // Each Latin-1 byte widens to exactly one UTF-16 code unit, and + // `dest` was sized as two bytes per source byte. + Ok(units * 2) + } + Conversion::FromUtf16 { to } => { + codecs::from_uchars(to, source, dest).ok_or(TranscodeError::UnableToTranscode) + } + Conversion::Utf16FromUtf8 => { + // `dest` was sized as two bytes per estimated code unit. + let expected_units = dest.len() / 2; + let units = codecs::convert_utf8_to_utf16le(source, dest); + // simdutf returns 0 for invalid UTF-8 input. + if units == 0 { + return Err(TranscodeError::UnableToTranscode); + } + if units != expected_units { + return Err(TranscodeError::Utf16LengthMismatch); + } + Ok(dest.len()) + } + Conversion::Utf8FromUtf16 => { + let expected_bytes = dest.len(); + let written = codecs::convert_utf16le_to_utf8(source, dest); + // simdutf returns 0 for invalid input, which fails this check + // because `expected_bytes` is nonzero here. The C++ + // `TranscodeUTF8FromUTF16` checks for 0 only *after* requiring + // equality with the (nonzero) estimate, so that branch is dead + // there too: a simdutf failure surfaces as a length mismatch, + // not "Unable to transcode buffer", unlike every other pair. + if written != expected_bytes { + return Err(TranscodeError::Utf8LengthMismatch); + } + Ok(written) + } + } + } + + /// ICU `ucnv_convertEx` between two converters, sized at `to`'s maximum + /// bytes per character. + fn convert_ex(source: &'a [u8], from: Encoding, to: Encoding) -> Result { + let to_conv = Converter::open(to)?; + to_conv.set_subst_chars(&"?".repeat(to_conv.min_char_size()))?; + let from_conv = Converter::open(from)?; + + let dest_len = source + .len() + .checked_mul(to_conv.max_char_size()) + .ok_or(TranscodeError::SourceBufferTooLarge)?; + if dest_len > ISOLATE_LIMIT { + return Err(TranscodeError::SourceBufferTooLarge); + } + + Ok(Self { + source, + conversion: Conversion::ConvertEx { + to: to_conv, + from: from_conv, + }, + dest_len, + }) + } + + /// simdutf widening of ASCII/Latin-1 into UTF-16. + /// + /// Taken for an `Ascii` source as well as a `Latin1` one, so source bytes + /// `0x80`-`0xFF` widen to U+0080-U+00FF instead of being substituted. + /// + /// No ICU converter is involved: the widening is purely arithmetic. + fn latin1_to_utf16(source: &'a [u8]) -> Result { + let dest_len = source + .len() + .checked_mul(2) + .ok_or(TranscodeError::SourceBufferTooLarge)?; + if dest_len > ISOLATE_LIMIT { + return Err(TranscodeError::SourceBufferTooLarge); + } + + Ok(Self { + source, + conversion: Conversion::Latin1ToUtf16, + dest_len, + }) + } + + /// ICU `ucnv_fromUChars` from UTF-16LE into `to`'s encoding, sized at + /// `to`'s maximum bytes per character. + fn from_utf16(source: &'a [u8], to: Encoding) -> Result { + if !source.len().is_multiple_of(2) { + return Err(TranscodeError::OddUtf16leInput); + } + + let to_conv = Converter::open(to)?; + to_conv.set_subst_chars(&"?".repeat(to_conv.min_char_size()))?; + + let dest_len = (source.len() / 2) + .checked_mul(to_conv.max_char_size()) + .ok_or(TranscodeError::BufferTooLarge)?; + if dest_len > ISOLATE_LIMIT { + return Err(TranscodeError::BufferTooLarge); + } + + Ok(Self { + source, + conversion: Conversion::FromUtf16 { to: to_conv }, + dest_len, + }) + } + + /// simdutf UTF-8 to UTF-16LE, sized from + /// `simdutf::utf16_length_from_utf8`. + /// + /// That estimate is zero for some non-empty inputs -- a source of nothing + /// but UTF-8 continuation bytes, for instance -- which yields an empty + /// result rather than an error. + fn utf16_from_utf8(source: &'a [u8]) -> Result { + let expected_units = codecs::utf16_length_from_utf8(source); + if expected_units > ISOLATE_LIMIT { + return Err(TranscodeError::ExpectedUtf16LengthTooLarge); + } + let dest_len = expected_units + .checked_mul(2) + .ok_or(TranscodeError::ExpectedUtf16LengthTooLarge)?; + + Ok(Self { + source, + conversion: Conversion::Utf16FromUtf8, + dest_len, + }) + } + + /// simdutf UTF-16LE to UTF-8, sized from + /// `simdutf::utf8_length_from_utf16le`. + fn utf8_from_utf16(source: &'a [u8]) -> Result { + if !source.len().is_multiple_of(2) { + return Err(TranscodeError::OddUtf16leInput); + } + + let dest_len = codecs::utf8_length_from_utf16le(source); + if dest_len > ISOLATE_LIMIT { + return Err(TranscodeError::ExpectedUtf8LengthTooLarge); + } + + Ok(Self { + source, + conversion: Conversion::Utf8FromUtf16, + dest_len, + }) + } +} + +#[cfg(test)] +mod tests { + use jsg_test::Harness; + + use super::*; + + /// All four transcodable encodings, for exhaustively testing every pair. + const ENCODINGS: [Encoding; 4] = [ + Encoding::Ascii, + Encoding::Latin1, + Encoding::Utf8, + Encoding::Utf16Le, + ]; + + // `Harness::new()` initializes the V8 platform, which is what installs the + // embedded ICU data. ICU converter opens fail without it, even though + // these tests never create an isolate or run JavaScript. + fn init_icu() -> Harness { + Harness::new() + } + + /// Runs both transcode steps the way [`crate::transcode`] does -- size, + /// allocate, convert, narrow to the written length -- against a plain + /// `Vec` rather than a V8 backing store, and returns the written bytes. + fn transcode(source: &[u8], from: Encoding, to: Encoding) -> Result, TranscodeError> { + let transcoder = Transcoder::new(source, from, to)?; + let mut dest = vec![0u8; transcoder.dest_len()]; + let written = transcoder.transcode_into(&mut dest)?; + assert!( + written <= dest.len(), + "{from:?} -> {to:?} wrote {written} bytes into a {} byte buffer", + dest.len() + ); + dest.truncate(written); + Ok(dest) + } + + #[test] + fn every_pair_round_trips_ascii_text() { + let _harness = init_icu(); + for &from in &ENCODINGS { + // UTF16LE source bytes must have even length; every other encoding + // is happy with plain ASCII bytes. + let source: &[u8] = if from == Encoding::Utf16Le { + &[0x48, 0x00, 0x69, 0x00] // "Hi" as UTF-16LE code units. + } else { + b"Hi" + }; + for &to in &ENCODINGS { + let result = transcode(source, from, to); + assert!(result.is_ok(), "{from:?} -> {to:?} failed: {result:?}"); + } + } + } + + #[test] + fn every_pair_empty_input_is_empty_output() { + let _harness = init_icu(); + for &from in &ENCODINGS { + for &to in &ENCODINGS { + let result = transcode(&[], from, to); + assert_eq!( + result.as_deref(), + Ok([].as_slice()), + "{from:?} -> {to:?} on empty input should be empty, got {result:?}" + ); + } + } + } + + #[test] + fn identity_pairs_round_trip() { + let _harness = init_icu(); + for &encoding in &ENCODINGS { + let source = b"identity"; + let result = transcode(source, encoding, encoding).unwrap(); + if encoding == Encoding::Utf16Le { + // `source` is treated as raw UTF-16LE code units, so identity is + // not a byte-for-byte passthrough of ASCII text; just check it + // succeeds (covered by `every_pair_round_trips_ascii_text`). + continue; + } + assert_eq!(result, source); + } + } + + #[test] + fn unmappable_characters_become_question_marks() { + let _harness = init_icu(); + // '☕' (U+2615, HOT BEVERAGE) has no representation in ASCII or Latin-1. + let source = "☕".as_bytes(); + assert_eq!( + transcode(source, Encoding::Utf8, Encoding::Ascii).unwrap(), + b"?" + ); + assert_eq!( + transcode(source, Encoding::Utf8, Encoding::Latin1).unwrap(), + b"?" + ); + } + + #[test] + fn ascii_to_utf16le_widens_high_bytes_via_latin1_path() { + let _harness = init_icu(); + // ASCII -> UTF16LE takes the Latin-1 path, so a high byte is widened to + // U+00FF rather than substituted with '?'. + let result = transcode(&[0xff], Encoding::Ascii, Encoding::Utf16Le).unwrap(); + assert_eq!(result, vec![0xff, 0x00]); + } + + #[test] + fn utf8_continuation_byte_only_input_yields_empty_utf16le() { + let _harness = init_icu(); + // The UTF8 -> UTF16LE size estimate is zero for input consisting only of + // continuation bytes, so the result is empty rather than an error, even + // though the input is non-empty. + let result = transcode(&[0x80], Encoding::Utf8, Encoding::Utf16Le).unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn invalid_utf8_to_utf16le_is_unable_to_transcode() { + let _harness = init_icu(); + let result = transcode(&[0x61, 0xc3], Encoding::Utf8, Encoding::Utf16Le); + assert_eq!(result, Err(TranscodeError::UnableToTranscode)); + } + + #[test] + fn odd_length_utf16le_to_utf8_is_rejected() { + let _harness = init_icu(); + let result = transcode(&[0x61], Encoding::Utf16Le, Encoding::Utf8); + assert_eq!(result, Err(TranscodeError::OddUtf16leInput)); + } + + #[test] + fn odd_length_utf16le_to_latin1_is_rejected() { + let _harness = init_icu(); + let result = transcode(&[0x61], Encoding::Utf16Le, Encoding::Latin1); + assert_eq!(result, Err(TranscodeError::OddUtf16leInput)); + } + + #[test] + fn unpaired_surrogate_utf16le_to_utf8() { + let _harness = init_icu(); + // U+D800, an unpaired high surrogate, encoded as UTF-16LE bytes. + let source = [0x00, 0xd8]; + // `simdutf::utf8_length_from_utf16le` does not validate, and reports the + // three bytes a replacement character would occupy, but + // `simdutf::convert_utf16le_to_utf8` rejects the input and writes + // nothing -- so the mismatch check is what surfaces the failure. + let result = transcode(&source, Encoding::Utf16Le, Encoding::Utf8); + assert_eq!(result, Err(TranscodeError::Utf8LengthMismatch)); + } + + #[test] + fn destination_size_mismatch_is_rejected() { + let _harness = init_icu(); + let transcoder = Transcoder::new(b"Hi", Encoding::Latin1, Encoding::Utf16Le).unwrap(); + let mut too_small = vec![0u8; transcoder.dest_len() - 1]; + assert_eq!( + transcoder.transcode_into(&mut too_small), + Err(TranscodeError::DestinationSizeMismatch) + ); + } +} diff --git a/src/rust/i18n/error.rs b/src/rust/i18n/error.rs new file mode 100644 index 00000000000..ad51062882e --- /dev/null +++ b/src/rust/i18n/error.rs @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The error type reported by [`crate::dispatch`], and its conversion into the +//! JavaScript `Error` the caller of `node:buffer`'s `transcode()` sees. + +use thiserror::Error; + +/// A failed transcode. +/// +/// The messages of the variants that have a C++ counterpart match the +/// corresponding `JSG_REQUIRE` / `JSG_FAIL_REQUIRE` string in +/// `workerd::api::node::i18n::transcode` (`src/workerd/api/node/i18n.c++`) +/// verbatim, so gate-on and gate-off are indistinguishable to JavaScript. +/// Do not reword them. +/// +/// `"Invalid encoding passed to transcode"` has no variant here: the C++ +/// `fromImpl` conversion raises it before the Rust entry point is reached, +/// since the bridge `Encoding` enum can only name the four transcodable +/// encodings. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TranscodeError { + #[error("Source buffer is too large to transcode")] + SourceBufferTooLarge, + #[error("Buffer is too large to transcode")] + BufferTooLarge, + #[error("Expected UTF-16le length is too large to transcode")] + ExpectedUtf16LengthTooLarge, + #[error("Expected UTF-8 length is too large to transcode")] + ExpectedUtf8LengthTooLarge, + #[error("UTF-16le input size should be multiple of 2")] + OddUtf16leInput, + #[error("Expected UTF16 length mismatch")] + Utf16LengthMismatch, + #[error("Expected UTF8 length mismatch")] + Utf8LengthMismatch, + #[error("Unable to transcode buffer")] + UnableToTranscode, + #[error("Failed to initialize converter")] + ConverterOpenFailed, + #[error("Setting ICU substitute characters failed")] + SetSubstituteCharsFailed, + // The remaining variants report broken internal invariants rather than bad + // input, and so have no C++ counterpart to match. + #[error("Invalid encoding passed to transcode")] + InvalidEncoding, + #[error("Destination buffer size does not match the prepared transcode")] + DestinationSizeMismatch, + #[error("Failed to allocate transcode destination buffer")] + AllocationFailed, +} + +impl From for jsg::Error { + fn from(value: TranscodeError) -> Self { + // All of these are plain JS `Error`s, matching the + // `JSG_REQUIRE(..., Error, ...)` calls they replace. + Self::new_error(value.to_string()) + } +} diff --git a/src/rust/i18n/lib.rs b/src/rust/i18n/lib.rs new file mode 100644 index 00000000000..662494016e4 --- /dev/null +++ b/src/rust/i18n/lib.rs @@ -0,0 +1,215 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Rust port of `workerd::api::node::i18n::transcode` +//! (`src/workerd/api/node/i18n.c++`), the engine behind `node:buffer`'s +//! `transcode()`. Selected at runtime by the `NODEJS_I18N_RUST` autogate; when +//! the gate is off, the C++ implementation is used instead. The two paths are +//! byte-for-byte and error-message identical by construction: [`dispatch`] +//! ports the C++ dispatch/sizing/truncation logic to Rust, while [`codecs`] +//! calls the exact same ICU and simdutf primitives the C++ path uses. +//! +//! ICU is bound by `rust_icu_sys`, configured for the Chromium ICU version +//! already linked into workerd. The simdutf primitives are bound directly by +//! the [`ffi`] bridge. Matching the C++ path's behaviour is a matter of calling +//! the same codecs the same way, not of sharing code with it. + +use jsg::Lock; +use jsg::v8; + +mod codecs; +mod dispatch; +mod error; + +use crate::dispatch::Transcoder; +use crate::error::TranscodeError; + +#[cxx::bridge(namespace = "workerd::rust::i18n")] +mod ffi { + /// The four encodings `i18n::transcode` supports. Mirrors the + /// transcodable subset of `workerd::api::node::Encoding` + /// (`src/workerd/api/node/i18n.h`). `src/workerd/api/node/i18n.c++` maps + /// into this type through a `fromImpl` overload that rejects the + /// non-transcodable `BASE64`, `BASE64URL`, and `HEX` variants before ever + /// calling into Rust. + #[derive(Debug, PartialEq, Eq, Copy, Clone)] + #[repr(u8)] + enum Encoding { + Ascii, + Latin1, + Utf8, + Utf16Le, + } + + // simdutf + // + // Each of these names is an overload set: a raw-pointer overload plus + // `std::span` and constrained-template overloads. cxx binds a C++ function + // by initializing an exactly-typed function pointer with its address, which + // picks the raw-pointer overload by exact match. + #[namespace = "simdutf"] + unsafe extern "C++" { + include!("simdutf.h"); + + unsafe fn convert_latin1_to_utf16( + input: *const c_char, + length: usize, + utf16_output: *mut c_char16, + ) -> usize; + unsafe fn utf16_length_from_utf8(input: *const c_char, length: usize) -> usize; + unsafe fn convert_utf8_to_utf16le( + input: *const c_char, + length: usize, + utf16_output: *mut c_char16, + ) -> usize; + unsafe fn utf8_length_from_utf16le(input: *const c_char16, length: usize) -> usize; + unsafe fn convert_utf16le_to_utf8( + input: *const c_char16, + length: usize, + utf8_output: *mut c_char, + ) -> usize; + } + + #[namespace = "workerd::rust::jsg"] + unsafe extern "C++" { + include!("workerd/rust/jsg/ffi.h"); + include!("workerd/rust/jsg/v8.rs.h"); + + type Isolate = jsg::v8::ffi::Isolate; + type MaybeLocal = jsg::v8::ffi::MaybeLocal; + } + + extern "Rust" { + /// Transcodes `source` from `from_encoding` to `to_encoding`, matching + /// `workerd::api::node::i18n::transcode`. Returns a `MaybeLocal` + /// naming a `Uint8Array`, or an empty `MaybeLocal` with a JS exception + /// already scheduled on `isolate` if transcoding fails. + /// + /// # Safety + /// `isolate` must be a valid pointer to a locked `v8::Isolate` with an + /// active `HandleScope`. + unsafe fn transcode( + isolate: *mut Isolate, + source: &[u8], + from_encoding: Encoding, + to_encoding: Encoding, + ) -> MaybeLocal; + } +} + +/// # Safety +/// `isolate` must be a valid pointer to a locked `v8::Isolate` with an active +/// `HandleScope`. +unsafe fn transcode( + isolate: *mut ffi::Isolate, + source: &[u8], + from_encoding: ffi::Encoding, + to_encoding: ffi::Encoding, +) -> ffi::MaybeLocal { + // SAFETY: forwarded from this function's own safety contract. The adapter + // owns the raw-isolate, exception, and local-handle FFI transitions; the + // feature callback operates only on safe JSG types. + unsafe { + v8::run_ffi_callback(isolate, |lock| { + transcode_impl(lock, source, from_encoding, to_encoding) + }) + } +} + +/// Transcodes `source` into a freshly allocated `Uint8Array`. +/// +/// The conversion writes straight into the V8 `ArrayBuffer`'s backing store. +/// There is no intermediate `Vec` and no copy: [`Transcoder`] reports the +/// destination size up front, the buffer is allocated at exactly that size, +/// and the returned view is narrowed to the bytes actually written. +fn transcode_impl<'a>( + lock: &'a mut Lock, + source: &[u8], + from_encoding: ffi::Encoding, + to_encoding: ffi::Encoding, +) -> Result, TranscodeError> { + let transcoder = Transcoder::new(source, from_encoding, to_encoding)?; + + // Every byte of the buffer is either written by the conversion or excluded + // from the returned view, so zeroing it first would be wasted work. + let mut buffer = v8::ArrayBuffer::new_with_mode( + lock, + transcoder.dest_len(), + v8::ffi::BackingStoreInitializationMode::Uninitialized, + ) + .ok_or(TranscodeError::AllocationFailed)?; + + let written = { + // SAFETY: `buffer` was created immediately above and has not been + // handed to JavaScript or aliased by another handle, so this is the + // only live reference into its backing store. `&mut Lock` is borrowed + // for the whole of `dest`, so no JavaScript can run and detach the + // buffer meanwhile. + let dest = unsafe { buffer.as_mut_slice(lock) }; + transcoder.transcode_into(dest)? + }; + + Ok(v8::Uint8Array::from_buffer(lock, &buffer, 0, written)) +} + +#[cfg(test)] +mod tests { + use jsg_test::Harness; + + use super::*; + + /// Exercises the full V8 path: allocate, convert into the backing store, + /// narrow the view. `dispatch.rs` covers conversion behaviour itself; this + /// checks the parts that only exist once V8 is involved. + #[test] + fn transcodes_into_a_narrowed_uint8_array() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // '☕' is three UTF-8 bytes and transcodes to the single byte "?" + // in ASCII, so the destination is allocated at 3 bytes and the + // returned view must be narrowed to 1. + let source = "☕".as_bytes(); + let array = + transcode_impl(lock, source, ffi::Encoding::Utf8, ffi::Encoding::Ascii).unwrap(); + assert_eq!(array.len(), 1); + assert_eq!(array.as_slice(), b"?"); + Ok(()) + }); + } + + #[test] + fn empty_input_yields_an_empty_uint8_array() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + let array = + transcode_impl(lock, &[], ffi::Encoding::Utf8, ffi::Encoding::Utf16Le).unwrap(); + assert!(array.is_empty()); + Ok(()) + }); + } + + #[test] + fn full_length_result_is_not_narrowed() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // Latin-1 -> UTF-16LE widens every byte to exactly two, so the + // conversion fills the destination exactly. + let array = + transcode_impl(lock, b"Hi", ffi::Encoding::Latin1, ffi::Encoding::Utf16Le).unwrap(); + assert_eq!(array.as_slice(), &[0x48, 0x00, 0x69, 0x00]); + Ok(()) + }); + } + + #[test] + fn failure_surfaces_as_an_error() { + let harness = Harness::new(); + harness.run_in_context(|lock, _ctx| { + // Odd-length UTF-16LE input is rejected before any allocation. + let result = transcode_impl(lock, &[0x61], ffi::Encoding::Utf16Le, ffi::Encoding::Utf8); + assert!(result.is_err()); + Ok(()) + }); + } +} diff --git a/src/rust/jsg/ffi-inl.h b/src/rust/jsg/ffi-inl.h index 70eb9af97bd..9cc65aee465 100644 --- a/src/rust/jsg/ffi-inl.h +++ b/src/rust/jsg/ffi-inl.h @@ -50,6 +50,12 @@ inline MaybeLocal maybe_local_to_ffi(v8::MaybeLocal value) { return MaybeLocal{result}; } +template +inline v8::MaybeLocal maybe_local_from_ffi(MaybeLocal&& value) { + auto ptr_void = reinterpret_cast(&value.ptr); + return *reinterpret_cast*>(ptr_void); +} + // Global // // ffi::Global stores only the strong v8::Global in `ptr`. diff --git a/src/rust/jsg/ffi.c++ b/src/rust/jsg/ffi.c++ index 1869c65d7d0..2b4a84e217e 100644 --- a/src/rust/jsg/ffi.c++ +++ b/src/rust/jsg/ffi.c++ @@ -779,6 +779,15 @@ DEFINE_TYPED_ARRAY_UNWRAP(biguint64_array, BigUint64Array, uint64_t) } // Local +Local uint8_array_from_buffer( + Isolate* isolate, const Local& buffer, size_t byte_offset, size_t length) { + auto arrayBuffer = local_as_ref_from_ffi(buffer); + KJ_REQUIRE( + byte_offset <= arrayBuffer->ByteLength() && length <= arrayBuffer->ByteLength() - byte_offset, + "Uint8Array view is out of bounds of its ArrayBuffer"); + return to_ffi(v8::Uint8Array::New(arrayBuffer, byte_offset, length)); +} + size_t local_typed_array_length(Isolate* isolate, const Local& array) { return local_as_ref_from_ffi(array)->Length(); } diff --git a/src/rust/jsg/ffi.h b/src/rust/jsg/ffi.h index 989a7e0de63..aacdffc1a28 100644 --- a/src/rust/jsg/ffi.h +++ b/src/rust/jsg/ffi.h @@ -207,6 +207,13 @@ Local local_new_float32_array(Isolate* isolate, const float* data, size_t length Local local_new_float64_array(Isolate* isolate, const double* data, size_t length); Local local_new_bigint64_array(Isolate* isolate, const int64_t* data, size_t length); Local local_new_biguint64_array(Isolate* isolate, const uint64_t* data, size_t length); +// Creates a Uint8Array view over an existing ArrayBuffer, without copying. Unlike +// local_new_uint8_array, which allocates a backing store and memcpys into it, this +// lets a caller allocate the buffer up front, write into it directly, and only then +// wrap the written prefix in a view. `byte_offset + length` must be within +// `buffer`'s byte length. +Local uint8_array_from_buffer( + Isolate* isolate, const Local& buffer, size_t byte_offset, size_t length); size_t local_typed_array_length(Isolate* isolate, const Local& array); // Returns a raw pointer to the underlying ArrayBuffer's data (without byte offset). // Use local_typed_array_byte_offset to compute the start of this view's data. diff --git a/src/rust/jsg/v8.rs b/src/rust/jsg/v8.rs index 61bba363cca..8b5bbc13aac 100644 --- a/src/rust/jsg/v8.rs +++ b/src/rust/jsg/v8.rs @@ -404,6 +404,12 @@ pub mod ffi { ) -> Result>; // Local + pub unsafe fn uint8_array_from_buffer( + isolate: *mut Isolate, + buffer: &Local, + byte_offset: usize, + length: usize, + ) -> Local; pub unsafe fn local_typed_array_length(isolate: *mut Isolate, array: &Local) -> usize; pub unsafe fn local_typed_array_buffer_data(isolate: *mut Isolate, array: &Local) -> usize; pub unsafe fn local_typed_array_byte_offset(isolate: *mut Isolate, array: &Local) -> usize; @@ -2318,6 +2324,47 @@ impl_typed_array!(BigUint64Array, u64, local_biguint64_array_get); // Uint8ClampedArray has the same element type as Uint8Array; clamping is a write-side JS concern. impl_typed_array!(Uint8ClampedArray, u8, local_uint8clamped_array_get); +impl Uint8Array { + /// Creates a `Uint8Array` viewing `length` bytes of `buffer`, starting at + /// `byte_offset`. + /// + /// Zero-copy, unlike [`Vec::to_js`](crate::ToJS), which allocates a fresh + /// backing store and copies into it. Producing a `Uint8Array` from bytes computed + /// in Rust therefore does not require an intermediate `Vec`: allocate the buffer + /// with [`ArrayBuffer::new_with_mode`], fill it through + /// [`Local::::as_mut_slice`], and wrap the written prefix here. + /// + /// # Panics + /// + /// Panics if `byte_offset + length` exceeds `buffer`'s byte length. The + /// check is repeated in C++, but failing it here keeps the failure a Rust + /// panic rather than a `kj::Exception` thrown across the bridge, which + /// this function's signature cannot carry. + pub fn from_buffer<'a>( + lock: &mut crate::Lock, + buffer: &Local<'_, ArrayBuffer>, + byte_offset: usize, + length: usize, + ) -> Local<'a, Self> { + let byte_length = buffer.byte_length(); + assert!( + byte_offset <= byte_length && length <= byte_length - byte_offset, + "Uint8Array view [{byte_offset}, {byte_offset}+{length}) is out of bounds \ + of its {byte_length}-byte ArrayBuffer" + ); + let isolate = lock.isolate(); + // SAFETY: Lock guarantees the isolate is locked and a HandleScope is active; + // `buffer` is a live handle to an ArrayBuffer. The bounds precondition is + // checked on the C++ side. + unsafe { + Local::from_ffi( + isolate, + ffi::uint8_array_from_buffer(isolate.as_ffi(), &buffer.handle, byte_offset, length), + ) + } + } +} + // ============================================================================= // `String`-specific implementations // ============================================================================= @@ -2483,6 +2530,43 @@ impl<'a, T> From>> for MaybeLocal<'a, T> { } } +/// Runs a synchronous callback entered from C++ with a valid, locked V8 isolate. +/// +/// The callback receives a safe [`Lock`] and returns a local handle. On success, +/// the handle is transferred back across the CXX bridge as a [`ffi::MaybeLocal`]. +/// On failure, the error is scheduled as a JavaScript exception and an empty +/// `MaybeLocal` is returned. +/// +/// This centralizes the raw-isolate and local-handle ownership transitions for +/// C++ entry points implemented in Rust. The callback itself contains no FFI +/// safety obligations. +/// +/// # Safety +/// +/// `isolate` must point to a live V8 isolate locked by the current thread, and +/// the C++ caller must keep its active `HandleScope` alive until it consumes the +/// returned handle. +pub unsafe fn run_ffi_callback(isolate: *mut ffi::Isolate, callback: F) -> ffi::MaybeLocal +where + E: Into, + F: for<'a> FnOnce(&'a mut Lock) -> Result, E>, +{ + // SAFETY: forwarded from this function's safety contract. + let mut lock = unsafe { Lock::from_isolate_ptr(isolate) }; + let result = callback(&mut lock).map(|local| { + // SAFETY: the callback's return lifetime is tied to `lock`, whose + // isolate is the one supplied by the active C++ HandleScope. + unsafe { local.into_ffi() } + }); + match result { + Ok(local) => ffi::MaybeLocal { ptr: local.ptr }, + Err(error) => { + lock.throw_exception(&error.into()); + ffi::MaybeLocal { ptr: 0 } + } + } +} + impl Local<'_, String> { // Instance methods — correspond to `v8::String` member functions diff --git a/src/workerd/api/node/BUILD.bazel b/src/workerd/api/node/BUILD.bazel index 2fca680d678..25c7cb938e1 100644 --- a/src/workerd/api/node/BUILD.bazel +++ b/src/workerd/api/node/BUILD.bazel @@ -96,6 +96,7 @@ wd_cc_library( ], implementation_deps = [ "//src/rust/cxx-integration", + "//src/rust/i18n", "//src/rust/net", "@ada-url", "@nbytes", diff --git a/src/workerd/api/node/i18n.c++ b/src/workerd/api/node/i18n.c++ index a895a7c72c0..1b270023286 100644 --- a/src/workerd/api/node/i18n.c++ +++ b/src/workerd/api/node/i18n.c++ @@ -8,7 +8,11 @@ #include "simdutf.h" #include +#include +#include +#include +#include #include #include #include @@ -18,6 +22,40 @@ namespace workerd::api::node { +namespace rust_i18n = ::workerd::rust::i18n; + +// Maps the C++ Encoding to the Rust bridge enum, following the kj-rs +// convert.h idiom (see `NODEJS_EXCEPTIONS_RUST`'s equivalent in +// exceptions.c++), so callers use `kj::from(value)`. It +// is `static` (rather than in an anonymous namespace) because Clang's ADL +// does not consider unnamed-namespace functions, and ADL is how +// `kj::from` locates this overload. The switch has no `default:` arm +// so that enum drift between the two `Encoding` types is a compile error. +// +// `BASE64`, `BASE64URL`, and `HEX` are not transcodable (see +// `i18n::canBeTranscoded`); rejecting them here, for both `fromEncoding` and +// `toEncoding`, is a deliberate divergence from the C++ dispatch below, which +// only checks `fromEncoding` and would reach `KJ_UNREACHABLE` for a +// non-transcodable `toEncoding`. Neither is reachable from JavaScript because +// `BufferUtil::transcode` validates both encodings first. +static rust_i18n::Encoding fromImpl(rust_i18n::Encoding*, Encoding encoding) { + switch (encoding) { + case Encoding::ASCII: + return rust_i18n::Encoding::Ascii; + case Encoding::LATIN1: + return rust_i18n::Encoding::Latin1; + case Encoding::UTF8: + return rust_i18n::Encoding::Utf8; + case Encoding::UTF16LE: + return rust_i18n::Encoding::Utf16Le; + case Encoding::BASE64: + case Encoding::BASE64URL: + case Encoding::HEX: + JSG_FAIL_REQUIRE(Error, "Invalid encoding passed to transcode"); + } + KJ_UNREACHABLE; +} + namespace i18n { namespace { @@ -240,6 +278,16 @@ void Converter::setSubstituteChars(kj::StringPtr sub) { jsg::JsUint8Array transcode( jsg::Lock& js, kj::ArrayPtr source, Encoding fromEncoding, Encoding toEncoding) { + if (util::Autogate::isEnabled(util::AutogateKey::NODEJS_I18N_RUST)) { + auto rustFrom = kj::from(fromEncoding); + auto rustTo = kj::from(toEncoding); + auto maybeLocal = + rust_i18n::transcode(js.v8Isolate, source.as(), rustFrom, rustTo); + auto local = + jsg::check(::workerd::rust::jsg::maybe_local_from_ffi(kj::mv(maybeLocal))); + return jsg::JsUint8Array(local); + } + TranscodeImpl transcode_function = &TranscodeDefault; switch (fromEncoding) { case Encoding::ASCII: diff --git a/src/workerd/util/autogate.h b/src/workerd/util/autogate.h index 99ddb48594b..46ff5e568a6 100644 --- a/src/workerd/util/autogate.h +++ b/src/workerd/util/autogate.h @@ -134,7 +134,12 @@ namespace workerd::util { never enters JavaScript and byte-budgets its un-yielded work, while pumpToImpl() still \ suspends through the event loop on every iteration (only the write suspension is elided), \ leaving the JS-visible pull() ordering unchanged. */ \ - V(STREAM_CONTROLLER_SYNC_FAST_PATHS) + V(STREAM_CONTROLLER_SYNC_FAST_PATHS) \ + /* When enabled, the Node.js `i18n` transcode implementation (api::node \ + i18n::transcode) is provided by the Rust implementation (src/rust/i18n) \ + instead of the C++ implementation. The C++ implementation is retained \ + for rollback.*/ \ + V(NODEJS_I18N_RUST) // clang-format on // --------------------------------------------------------------------------------------