From c250cc9bc41f8e3986b9b0165fb75eea40ae3a5d Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 3 Sep 2026 18:50:36 -0700 Subject: [PATCH 1/4] Add optional Hugging Face .tok tokenizer backend --- CMakeLists.txt | 9 + cmake/tokenizers-config.cmake.in | 19 + .../pytorch/tokenizers/rust_hf_tokenizer.h | 56 + rust_tokenizer/.gitignore | 1 + rust_tokenizer/CMakeLists.txt | 79 + rust_tokenizer/Cargo.lock | 1456 +++++++++++++++++ rust_tokenizer/Cargo.toml | 33 + rust_tokenizer/README.md | 45 + rust_tokenizer/binary-size.md | 14 + rust_tokenizer/src/lib.rs | 443 +++++ src/rust_hf_tokenizer.cpp | 375 +++++ 11 files changed, 2530 insertions(+) create mode 100644 include/pytorch/tokenizers/rust_hf_tokenizer.h create mode 100644 rust_tokenizer/.gitignore create mode 100644 rust_tokenizer/CMakeLists.txt create mode 100644 rust_tokenizer/Cargo.lock create mode 100644 rust_tokenizer/Cargo.toml create mode 100644 rust_tokenizer/README.md create mode 100644 rust_tokenizer/binary-size.md create mode 100644 rust_tokenizer/src/lib.rs create mode 100644 src/rust_hf_tokenizer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 62ca12cf..f1eaa8fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,11 @@ project(Tokenizers) option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) +option( + TOKENIZERS_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face .tok tokenizer backend" OFF +) +option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" OFF) option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF ) @@ -201,6 +206,10 @@ endif() # Installation rules include(GNUInstallDirs) +if(TOKENIZERS_BUILD_HF_RUST_TOKENIZER) + add_subdirectory(rust_tokenizer) +endif() + if(NOT TOKENIZERS_BUILD_PYTHON) # Install the library and its dependencies install( diff --git a/cmake/tokenizers-config.cmake.in b/cmake/tokenizers-config.cmake.in index 1e9f87b2..6a075dd5 100644 --- a/cmake/tokenizers-config.cmake.in +++ b/cmake/tokenizers-config.cmake.in @@ -25,6 +25,25 @@ endif() find_dependency(re2 REQUIRED) find_dependency(absl REQUIRED) +# The optional .tok backend is a C++ archive with a Rust static-library +# dependency. Define the latter before importing the exported targets when the +# archive is present. Merely finding the package does not link either target. +if(WIN32) + set(_TOKENIZERS_HF_FFI_NAME "tokenizers_hf_ffi.lib") +else() + set(_TOKENIZERS_HF_FFI_NAME "libtokenizers_hf_ffi.a") +endif() +set(_TOKENIZERS_HF_FFI "${TOKENIZERS_LIBDIR}/${_TOKENIZERS_HF_FFI_NAME}") +if(EXISTS "${_TOKENIZERS_HF_FFI}" AND NOT TARGET tokenizers_hf_ffi) + if(UNIX AND NOT APPLE) + find_dependency(Threads) + endif() + add_library(tokenizers_hf_ffi STATIC IMPORTED) + set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION "${_TOKENIZERS_HF_FFI}" + ) +endif() + # Include the exported targets file include("${CMAKE_CURRENT_LIST_DIR}/tokenizers-targets.cmake") diff --git a/include/pytorch/tokenizers/rust_hf_tokenizer.h b/include/pytorch/tokenizers/rust_hf_tokenizer.h new file mode 100644 index 00000000..b75e8fc7 --- /dev/null +++ b/include/pytorch/tokenizers/rust_hf_tokenizer.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace tokenizers { + +class RustHFTokenizer final : public Tokenizer { + public: + RustHFTokenizer(); + ~RustHFTokenizer() override; + + Error load(const std::string& tokenizer_path) override; + Result id_to_piece(uint64_t token) const override; + Result piece_to_id(const std::string& text) const override; + Result> encode( + const std::string& input, + int8_t bos = 0, + int8_t eos = 0) const override; + Result decode( + uint64_t prev_token, + uint64_t token, + bool skip_special_tokens = false) const override; + + private: + using TokenMap = detail::StringIntegerMap<>; + + struct RustHandleDeleter { + void operator()(void* handle) const; + }; + + Error load_metadata(const void* handle); + + std::unique_ptr handle_; + std::optional token_map_; + std::optional added_token_map_; + std::unordered_set special_token_ids_; + bool byte_level_ = false; +}; + +} // namespace tokenizers diff --git a/rust_tokenizer/.gitignore b/rust_tokenizer/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/rust_tokenizer/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/rust_tokenizer/CMakeLists.txt b/rust_tokenizer/CMakeLists.txt new file mode 100644 index 00000000..ee219656 --- /dev/null +++ b/rust_tokenizer/CMakeLists.txt @@ -0,0 +1,79 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +find_program(CARGO_EXECUTABLE cargo REQUIRED) + +set(_rust_profile release) +if(TOKENIZERS_OPTIMIZE_SIZE) + set(_rust_profile minsize) +endif() + +set(_cargo_target_dir ${CMAKE_CURRENT_BINARY_DIR}/cargo-target) +string( + CONCAT + _rust_library_name + ${CMAKE_STATIC_LIBRARY_PREFIX} + tokenizers_hf_ffi + ${CMAKE_STATIC_LIBRARY_SUFFIX} +) +set(_rust_library ${_cargo_target_dir}/${_rust_profile}/${_rust_library_name}) +file(GLOB_RECURSE _rust_sources CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/src/*.rs +) + +add_custom_command( + OUTPUT ${_rust_library} + COMMAND + ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_cargo_target_dir} + CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic + ${CARGO_EXECUTABLE} build --locked --manifest-path + ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile ${_rust_profile} + DEPENDS ${_rust_sources} ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml + ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM +) + +add_custom_target(tokenizers_hf_ffi_build DEPENDS ${_rust_library}) +add_library(tokenizers_hf_ffi STATIC IMPORTED GLOBAL) +set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION ${_rust_library} +) +add_dependencies(tokenizers_hf_ffi tokenizers_hf_ffi_build) + +add_library(tokenizers_hf_rust_tokenizer STATIC ../src/rust_hf_tokenizer.cpp) +add_library(tokenizers::hf_rust_tokenizer ALIAS tokenizers_hf_rust_tokenizer) +set_target_properties( + tokenizers_hf_rust_tokenizer PROPERTIES EXPORT_NAME hf_rust_tokenizer +) +target_include_directories( + tokenizers_hf_rust_tokenizer + PUBLIC + $ + $ +) +target_compile_features(tokenizers_hf_rust_tokenizer PUBLIC cxx_std_17) +target_link_libraries( + tokenizers_hf_rust_tokenizer + PRIVATE tokenizers_hf_ffi +) + +if(APPLE) + target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE iconv) +elseif(UNIX) + find_package(Threads REQUIRED) + target_link_libraries( + tokenizers_hf_rust_tokenizer + PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m rt util + ) +endif() + +install( + TARGETS tokenizers_hf_rust_tokenizer + EXPORT tokenizers-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +install(FILES ${_rust_library} DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/rust_tokenizer/Cargo.lock b/rust_tokenizer/Cargo.lock new file mode 100644 index 00000000..87eca543 --- /dev/null +++ b/rust_tokenizer/Cargo.lock @@ -0,0 +1,1456 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary-chunks" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" + +[[package]] +name = "atomsplit" +version = "0.1.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cacheline-ef" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85" +dependencies = [ + "common_traits", + "mem_dbg 0.3.4", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "common_traits" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062" +dependencies = [ + "anyhow", + "half", + "impl-tools", +] + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +dependencies = [ + "tk-encode", + "tk-serialization", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "impl-tools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" +dependencies = [ + "autocfg", + "impl-tools-lib", + "proc-macro-error2", + "syn 2.0.119", +] + +[[package]] +name = "impl-tools-lib" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + +[[package]] +name = "mem_dbg" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0" +dependencies = [ + "bitflags", + "mem_dbg-derive 0.2.1", +] + +[[package]] +name = "mem_dbg" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" +dependencies = [ + "bitflags", + "hashbrown", + "mem_dbg-derive 0.3.4", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "partition" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefetch-index" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_hash" +version = "2.0.2" +source = "git+https://github.com/ArthurZucker/PtrHash?rev=fff63a67eec9b48b693b0d3d2b253db78c0b9858#fff63a67eec9b48b693b0d3d2b253db78c0b9858" +dependencies = [ + "bitvec", + "cacheline-ef", + "colored", + "fastrand", + "fxhash", + "itertools 0.15.0", + "log", + "mem_dbg 0.4.4", + "prefetch-index", + "rand 0.10.2", + "rand_chacha 0.10.0", + "rdst", + "sucds", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdst" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +dependencies = [ + "arbitrary-chunks", + "block-pseudorand", + "criterion", + "partition", + "tikv-jemallocator", + "voracious_radix_sort", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "sucds" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd324eaa05be64f105ea5269bb8aabd70e5dd57fa5c673b167f451b07d6c0dcd" +dependencies = [ + "anyhow", + "num-traits", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tk-encode" +version = "0.23.2-dev.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" +dependencies = [ + "ahash", + "atomsplit", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "memchr", + "monostate", + "paste", + "ptr_hash", + "rand 0.9.5", + "regex", + "thiserror", + "tk-serialization", + "unicode-segmentation", + "unicode_categories", + "yada", +] + +[[package]] +name = "tk-serialization" +version = "0.23.2-dev.0" +source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yada" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust_tokenizer/Cargo.toml b/rust_tokenizer/Cargo.toml new file mode 100644 index 00000000..167d7de8 --- /dev/null +++ b/rust_tokenizer/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +name = "tokenizers_hf_ffi" +crate-type = ["staticlib"] + +[dependencies] +tk-encode = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", default-features = false } +tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5" } + +[dev-dependencies] +tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", features = ["write"] } + +# The `.tok` branch relies on PtrHash's dependency gates until they are +# available in a crates.io release. Pin the same revision rather than following +# the mutable upstream branch. +[patch.crates-io] +ptr_hash = { git = "https://github.com/ArthurZucker/PtrHash", rev = "fff63a67eec9b48b693b0d3d2b253db78c0b9858" } + +[profile.release] +codegen-units = 1 +lto = "fat" +opt-level = 3 +strip = true + +[profile.minsize] +inherits = "release" +opt-level = "z" +panic = "abort" diff --git a/rust_tokenizer/README.md b/rust_tokenizer/README.md new file mode 100644 index 00000000..63f82a04 --- /dev/null +++ b/rust_tokenizer/README.md @@ -0,0 +1,45 @@ +# Experimental Hugging Face Rust tokenizer backend + +This backend is opt-in and disabled by default. Enable it with +`-DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON`. When disabled, CMake does not enter +this directory, invoke Cargo, build the Rust archive, or compile the C++ +bridge, so the default tokenizer library is unchanged. ExecuTorch users can +enable the same target with both `-DEXECUTORCH_BUILD_EXTENSION_LLM=ON` and +`-DEXECUTORCH_BUILD_HF_RUST_TOKENIZER=ON`. +Use `-DTOKENIZERS_OPTIMIZE_SIZE=ON` for the Rust `minsize` profile; ExecuTorch +forwards `EXECUTORCH_OPTIMIZE_SIZE` to this option. + +When enabled, the LLM runner uses Hugging Face's parser-free inference pipeline +for `.tok` files while retaining the existing ExecuTorch `Tokenizer` interface. +A directory input uses `/tokenizer.tok`. JSON tokenizers continue +through the existing C++ `HFTokenizer`; all other tokenizer fallbacks are +unchanged. + +Create the artifact offline with the `tk-convert` tool from Hugging Face's +[`feat/tok-format`](https://github.com/huggingface/tokenizers/tree/feat/tok-format) +branch: + +```sh +cargo run --release --manifest-path tokenizers/Cargo.toml -p tk-convert -- \ + /path/to/tokenizer.json +``` + +This writes `/path/to/tokenizer.tok`. The v1 container supports BPE, Unigram, +WordPiece, and WordLevel models, but intentionally supports only the +normalizers and pre-tokenizers represented by the format. Conversion fails +instead of silently dropping unsupported behavior. + +This experiment currently supports host CMake builds. Android, Apple framework, +WASM, and Buck packaging still need explicit Rust target/toolchain integration. +The build requires Cargo and fetches the pinned Rust dependencies on its first +run. + +The Rust dependencies are pinned to Hugging Face tokenizers commit +`054bdf469b2cf416c0da953923d88c82f4d765b5` from `feat/tok-format`. Only +`tk-encode` and the zero-dependency `.tok` reader are linked; JSON conversion, +serde, training, and progress-bar code stay out of the runtime binary. The +Rust pipeline owns encoding. The C++ compatibility layer reads vocabulary and +special-token metadata from the same `.tok` image and applies byte-level +decoding when the format marks the model as byte-level. Other decoder chains +are not represented by `.tok` v1 and therefore retain raw-piece incremental +decode behavior. diff --git a/rust_tokenizer/binary-size.md b/rust_tokenizer/binary-size.md new file mode 100644 index 00000000..8ad6ab6e --- /dev/null +++ b/rust_tokenizer/binary-size.md @@ -0,0 +1,14 @@ +# Hugging Face `.tok` backend binary size + +Measured on Apple arm64 with Rust 1.98.1, `CMAKE_BUILD_TYPE=Release`, +`TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. The smoke binary +loads a GPT-2 `.tok`, encodes `Hello world`, performs vocabulary lookups, and +decodes both output tokens. + +| Configuration | Stripped | Gzipped | +|---|---:|---:| +| Default (`TOKENIZERS_BUILD_HF_RUST_TOKENIZER=OFF`) | 0 B added | 0 B added | +| Opt-in `.tok` backend | 591,040 B | 298,961 B | + +The OFF configuration exposes no Rust CMake target and produces no Cargo build +directory. diff --git a/rust_tokenizer/src/lib.rs b/rust_tokenizer/src/lib.rs new file mode 100644 index 00000000..23c68431 --- /dev/null +++ b/rust_tokenizer/src/lib.rs @@ -0,0 +1,443 @@ +use std::ffi::{CStr, c_char, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use tk_encode::pipeline::PipelineTokenizer; +use tk_serialization::{AddedEntry, Entry, TokFile, added_flag, kind}; + +struct Handle { + tokenizer: PipelineTokenizer, + // Keep the aligned `.tok` image alive. `PipelineTokenizer::from_tok` + // currently owns the structures it builds, but retaining the image makes + // the FFI safe if the upstream reader starts borrowing sections later. + file: TokFile, +} + +/// Create a tokenizer from a `.tok` path. +/// +/// # Safety +/// +/// `path` must be null or point to a valid NUL-terminated C string for the +/// duration of this call. A non-null return value must eventually be passed +/// exactly once to [`tokenizers_hf_destroy`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_void { + catch_unwind(AssertUnwindSafe(|| { + if path.is_null() { + return std::ptr::null_mut(); + } + let Ok(path) = unsafe { CStr::from_ptr(path) }.to_str() else { + return std::ptr::null_mut(); + }; + let Ok(file) = TokFile::open(path) else { + return std::ptr::null_mut(); + }; + let Ok(tokenizer) = PipelineTokenizer::from_tok(file.bytes()) else { + return std::ptr::null_mut(); + }; + Box::into_raw(Box::new(Handle { tokenizer, file })).cast() + })) + .unwrap_or(std::ptr::null_mut()) +} + +/// Encode one UTF-8 string into caller-owned token storage. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`]. `text` must address `text_len` readable +/// bytes unless `text_len` is zero. `output` must address `output_capacity` +/// writable `u32` values unless the capacity is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_encode( + opaque: *const c_void, + text: *const u8, + text_len: usize, + add_special_tokens: u8, + output: *mut u32, + output_capacity: usize, +) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || (text.is_null() && text_len != 0) { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let bytes = if text_len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(text, text_len) } + }; + let Ok(text) = std::str::from_utf8(bytes) else { + return -1; + }; + + let Ok(tokens) = handle.tokenizer.encode(text, add_special_tokens != 0) else { + return -1; + }; + let Ok(token_count) = isize::try_from(tokens.len()) else { + return -1; + }; + if tokens.len() > output_capacity { + return token_count; + } + if !tokens.is_empty() && output.is_null() { + return -1; + } + for (index, token) in tokens.iter().enumerate() { + unsafe { output.add(index).write(token.id) }; + } + token_count + })) + .unwrap_or(-1) +} + +/// Return the number of vocabulary and added-token records in the `.tok`. +/// +/// This is a record count rather than a vocabulary size because an added token +/// can deliberately reuse a model-vocabulary id. The C++ compatibility layer +/// removes those duplicates while constructing its lookup maps. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_count(opaque: *const c_void) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { + return -1; + }; + let Ok(added) = reader.section::(kind::ADDED_ENTRY) else { + return -1; + }; + vocab + .len() + .checked_add(added.len()) + .and_then(|count| isize::try_from(count).ok()) + .unwrap_or(-1) + })) + .unwrap_or(-1) +} + +/// Read one vocabulary record. Returned string bytes borrow `opaque` and stay +/// valid until the handle is destroyed. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`]. Every output argument must point to +/// writable storage of its declared type. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_at( + opaque: *const c_void, + index: usize, + id: *mut u32, + text: *mut *const u8, + text_len: *mut usize, + is_added: *mut u8, + is_special: *mut u8, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() + || id.is_null() + || text.is_null() + || text_len.is_null() + || is_added.is_null() + || is_special.is_null() + { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { + return -1; + }; + let (slab, start, len, token_id, added, special) = if index < vocab.len() { + let Ok(slab) = reader.require::(kind::VOCAB_SLAB) else { + return -1; + }; + let entry = vocab[index]; + (slab, entry.start, entry.len, entry.id, false, false) + } else { + let Ok(entries) = reader.section::(kind::ADDED_ENTRY) else { + return -1; + }; + let Some(entry) = entries.get(index - vocab.len()).copied() else { + return -1; + }; + let Ok(slab) = reader.section::(kind::ADDED_SLAB) else { + return -1; + }; + ( + slab, + entry.start, + entry.len, + entry.id, + true, + entry.flags & added_flag::SPECIAL != 0, + ) + }; + let Some(end) = (start as usize).checked_add(len as usize) else { + return -1; + }; + let Some(bytes) = slab.get(start as usize..end) else { + return -1; + }; + if std::str::from_utf8(bytes).is_err() { + return -1; + } + unsafe { + id.write(token_id); + text.write(bytes.as_ptr()); + text_len.write(bytes.len()); + is_added.write(u8::from(added)); + is_special.write(u8::from(special)); + } + 0 + })) + .unwrap_or(-1) +} + +/// Return the first prefix token (`suffix == 0`) or last suffix token +/// (`suffix != 0`) from the `.tok` post-processor. Returns 0 when present, 1 +/// when the requested side is empty, and -1 on error. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by +/// [`tokenizers_hf_create`] and `token` must point to a writable +/// `u32`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_post_token( + opaque: *const c_void, + suffix: u8, + token: *mut u32, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || token.is_null() { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Ok(reader) = handle.file.reader() else { + return -1; + }; + let Ok(tokens) = reader.section::(if suffix == 0 { + kind::POST_PREFIX + } else { + kind::POST_SUFFIX + }) else { + return -1; + }; + let value = if suffix == 0 { + tokens.first() + } else { + tokens.last() + }; + let Some(value) = value else { + return 1; + }; + unsafe { token.write(*value) }; + 0 + })) + .unwrap_or(-1) +} + +/// Return `.tok` config flags, or `u32::MAX` for an invalid handle/image. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_config_flags(opaque: *const c_void) -> u32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return u32::MAX; + } + let handle = unsafe { &*opaque.cast::() }; + handle + .file + .reader() + .map(|reader| reader.config.flags) + .unwrap_or(u32::MAX) + })) + .unwrap_or(u32::MAX) +} + +/// Destroy a tokenizer handle. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by +/// [`tokenizers_hf_create`] that has not previously been destroyed. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_destroy(opaque: *mut c_void) { + if !opaque.is_null() { + let _ = catch_unwind(AssertUnwindSafe(|| { + drop(unsafe { Box::from_raw(opaque.cast::()) }); + })); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use tk_serialization::{Config, Writer, model, pretok, strings}; + + fn test_image() -> Vec { + let config = Config { + model: model::WORDLEVEL, + model_param: 0, + pretok: pretok::NONE, + pretok_param: 0, + flags: 0, + _pad0: 0, + added_first: [1 << (b'<' & 63), 0, 0, 0], + }; + let vocab_slab = b"hello"; + let vocab = [ + Entry { + start: 0, + len: 5, + id: 0, + }, + Entry { + start: 5, + len: 5, + id: 1, + }, + ]; + let added_slab = b""; + let added = [AddedEntry { + start: 0, + len: 3, + id: 2, + flags: added_flag::SPECIAL, + }]; + let mut model_strings = Vec::new(); + strings::push(&mut model_strings, ""); + strings::push(&mut model_strings, ""); + strings::push(&mut model_strings, ""); + + let mut writer = Writer::new(); + writer.push_one(kind::CONFIG, &config); + writer.push(kind::VOCAB_SLAB, vocab_slab); + writer.push(kind::VOCAB_ENTRY, &vocab); + writer.push(kind::ADDED_SLAB, added_slab); + writer.push(kind::ADDED_ENTRY, &added); + writer.push(kind::POST_PREFIX, &[2u32]); + writer.push(kind::POST_SUFFIX, &[2u32]); + writer.push(kind::MODEL_STRINGS, &model_strings); + writer.finish() + } + + fn with_handle(test: impl FnOnce(*mut c_void)) { + let path = std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-{}-{}.tok", + std::process::id(), + std::thread::current().name().unwrap_or("test") + )); + std::fs::write(&path, test_image()).unwrap(); + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + let handle = unsafe { tokenizers_hf_create(c_path.as_ptr()) }; + assert!(!handle.is_null()); + test(handle); + unsafe { tokenizers_hf_destroy(handle) }; + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn loads_and_encodes_tok() { + with_handle(|handle| { + let text = b"hello"; + let required = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + 1, + std::ptr::null_mut(), + 0, + ) + }; + assert_eq!(required, 3); + let mut output = [0u32; 3]; + let written = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + 1, + output.as_mut_ptr(), + output.len(), + ) + }; + assert_eq!(written, 3); + assert_eq!(output, [2, 1, 2]); + }); + } + + #[test] + fn exposes_tok_metadata() { + with_handle(|handle| { + assert_eq!(unsafe { tokenizers_hf_token_count(handle) }, 3); + + let mut id = 0; + let mut text = std::ptr::null(); + let mut len = 0; + let mut added = 0; + let mut special = 0; + assert_eq!( + unsafe { + tokenizers_hf_token_at( + handle, + 2, + &mut id, + &mut text, + &mut len, + &mut added, + &mut special, + ) + }, + 0 + ); + assert_eq!(id, 2); + assert_eq!(unsafe { std::slice::from_raw_parts(text, len) }, b""); + assert_eq!((added, special), (1, 1)); + + let mut token = 0; + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 0, &mut token) }, + 0 + ); + assert_eq!(token, 2); + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 1, &mut token) }, + 0 + ); + assert_eq!(token, 2); + assert_eq!(unsafe { tokenizers_hf_config_flags(handle) }, 0); + }); + } + + #[test] + fn rejects_non_tok_input() { + let path = std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-invalid-{}.tok", + std::process::id() + )); + std::fs::write(&path, b"not a tok").unwrap(); + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + assert!(unsafe { tokenizers_hf_create(c_path.as_ptr()) }.is_null()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/src/rust_hf_tokenizer.cpp b/src/rust_hf_tokenizer.cpp new file mode 100644 index 00000000..7604eeb2 --- /dev/null +++ b/src/rust_hf_tokenizer.cpp @@ -0,0 +1,375 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +extern "C" { +void* tokenizers_hf_create(const char* path); +intptr_t tokenizers_hf_encode( + const void* handle, + const uint8_t* text, + size_t text_len, + uint8_t add_special_tokens, + uint32_t* output, + size_t output_capacity); +intptr_t tokenizers_hf_token_count(const void* handle); +int32_t tokenizers_hf_token_at( + const void* handle, + size_t index, + uint32_t* id, + const uint8_t** text, + size_t* text_len, + uint8_t* is_added, + uint8_t* is_special); +int32_t tokenizers_hf_post_token( + const void* handle, + uint8_t suffix, + uint32_t* token); +uint32_t tokenizers_hf_config_flags(const void* handle); +void tokenizers_hf_destroy(void* handle); +} + +namespace tokenizers { +namespace { + +// Mirrors tk_serialization::flag::BYTE_LEVEL. The `.tok` v1 format deliberately +// keeps these values stable as part of its on-disk schema. +constexpr uint32_t kByteLevelFlag = 1U << 2; + +std::optional byte_level_codepoint_to_byte(uint32_t codepoint) { + if ((codepoint >= 33 && codepoint <= 126) || + (codepoint >= 161 && codepoint <= 172) || + (codepoint >= 174 && codepoint <= 255)) { + return static_cast(codepoint); + } + if (codepoint < 256 || codepoint > 323) { + return std::nullopt; + } + const auto index = codepoint - 256; + if (index < 33) { + return static_cast(index); + } + if (index < 67) { + return static_cast(127 + index - 33); + } + return static_cast(173); +} + +std::string decode_byte_level(std::string_view piece) { + std::string decoded; + decoded.reserve(piece.size()); + for (size_t index = 0; index < piece.size();) { + const auto first = static_cast(piece[index]); + uint32_t codepoint = 0; + size_t length = 0; + if ((first & 0x80) == 0) { + codepoint = first; + length = 1; + } else if ((first & 0xE0) == 0xC0) { + codepoint = first & 0x1F; + length = 2; + } else if ((first & 0xF0) == 0xE0) { + codepoint = first & 0x0F; + length = 3; + } else if ((first & 0xF8) == 0xF0) { + codepoint = first & 0x07; + length = 4; + } else { + return std::string(piece); + } + if (length > piece.size() - index) { + return std::string(piece); + } + for (size_t offset = 1; offset < length; ++offset) { + const auto continuation = + static_cast(piece[index + offset]); + if ((continuation & 0xC0) != 0x80) { + return std::string(piece); + } + codepoint = (codepoint << 6) | (continuation & 0x3F); + } + const auto byte = byte_level_codepoint_to_byte(codepoint); + if (!byte) { + return std::string(piece); + } + decoded.push_back(static_cast(*byte)); + index += length; + } + return decoded; +} + +} // namespace + +RustHFTokenizer::RustHFTokenizer() : handle_(nullptr) {} + +RustHFTokenizer::~RustHFTokenizer() = default; + +void RustHFTokenizer::RustHandleDeleter::operator()(void* handle) const { + tokenizers_hf_destroy(handle); +} + +Error RustHFTokenizer::load(const std::string& path) { + initialized_ = false; + handle_.reset(); + token_map_.reset(); + added_token_map_.reset(); + special_token_ids_.clear(); + byte_level_ = false; + vocab_size_ = 0; + bos_tok_ = 0; + eos_tok_ = 0; + + std::string tokenizer_tok = path; + std::error_code fs_error; + if (fs::is_directory(path, fs_error)) { + const fs::path root(path); + tokenizer_tok = (root / "tokenizer.tok").string(); + } + if (fs_error || !fs::exists(tokenizer_tok, fs_error) || fs_error) { + return Error::LoadFailure; + } + + std::unique_ptr handle( + tokenizers_hf_create(tokenizer_tok.c_str())); + if (!handle) { + return Error::LoadFailure; + } + const auto metadata_error = load_metadata(handle.get()); + if (metadata_error != Error::Ok) { + return metadata_error; + } + + const auto flags = tokenizers_hf_config_flags(handle.get()); + if (flags == std::numeric_limits::max()) { + return Error::ParseFailure; + } + byte_level_ = (flags & kByteLevelFlag) != 0; + + handle_ = std::move(handle); + initialized_ = true; + return Error::Ok; +} + +Error RustHFTokenizer::load_metadata(const void* handle) { + const auto count = tokenizers_hf_token_count(handle); + if (count < 0 || + static_cast(count) > + static_cast(std::numeric_limits::max())) { + return Error::ParseFailure; + } + + struct TokenRecord { + std::string text; + uint64_t id; + bool added; + bool special; + }; + std::vector records; + records.reserve(static_cast(count)); + std::unordered_set added_ids; + for (size_t index = 0; index < static_cast(count); ++index) { + uint32_t id = 0; + const uint8_t* text = nullptr; + size_t text_len = 0; + uint8_t is_added = 0; + uint8_t is_special = 0; + if (tokenizers_hf_token_at( + handle, + index, + &id, + &text, + &text_len, + &is_added, + &is_special) != 0 || + (text == nullptr && text_len != 0)) { + return Error::ParseFailure; + } + records.push_back( + {std::string(reinterpret_cast(text), text_len), + id, + is_added != 0, + is_special != 0}); + if (is_added != 0) { + added_ids.insert(id); + if (is_special != 0) { + special_token_ids_.insert(id); + } + } + } + + std::vector> tokens; + std::vector> added_tokens; + std::vector bos_candidates; + std::vector eos_candidates; + tokens.reserve(records.size()); + added_tokens.reserve(added_ids.size()); + for (auto& record : records) { + if (record.special) { + if (record.text.find("bos") != std::string::npos || + record.text.find("begin") != std::string::npos) { + bos_candidates.push_back(record.id); + } + if (record.text.find("eos") != std::string::npos || + record.text.find("end") != std::string::npos) { + eos_candidates.push_back(record.id); + } + } + if (record.added) { + added_tokens.emplace_back(std::move(record.text), record.id); + } else if (added_ids.count(record.id) == 0) { + tokens.emplace_back(std::move(record.text), record.id); + } + } + auto added_map = TokenMap::create(added_tokens); + if (!added_map.ok()) { + return added_map.error(); + } + added_token_map_.emplace(std::move(*added_map)); + auto token_map = TokenMap::create(tokens); + if (!token_map.ok()) { + return token_map.error(); + } + token_map_.emplace(std::move(*token_map)); + vocab_size_ = static_cast( + token_map_->size() + added_token_map_->size()); + + uint32_t bos = 0; + uint32_t eos = 0; + const auto bos_status = + tokenizers_hf_post_token(handle, 0, &bos); + const auto eos_status = + tokenizers_hf_post_token(handle, 1, &eos); + if (bos_status < 0 || eos_status < 0) { + return Error::ParseFailure; + } + if (bos_status == 0) { + bos_tok_ = bos; + } + if (eos_status == 0) { + eos_tok_ = eos; + } + bool bos_found = bos_status == 0; + bool eos_found = eos_status == 0; + if (!bos_found || !eos_found) { + if (!bos_found && bos_candidates.size() == 1) { + bos_tok_ = bos_candidates.front(); + bos_found = true; + } + if (!eos_found && eos_candidates.size() == 1) { + eos_tok_ = eos_candidates.front(); + eos_found = true; + } + } + if (bos_found && !eos_found) { + eos_tok_ = bos_tok_; + } else if (!bos_found && eos_found) { + bos_tok_ = eos_tok_; + } + return Error::Ok; +} + +Result RustHFTokenizer::id_to_piece(uint64_t token) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto piece = token_map_->tryGetString(token)) { + return std::string(*piece); + } + if (auto piece = added_token_map_->tryGetString(token)) { + return std::string(*piece); + } + return Error::OutOfRange; +} + +Result RustHFTokenizer::piece_to_id(const std::string& text) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto id = token_map_->tryGetInteger(text)) { + return *id; + } + if (auto id = added_token_map_->tryGetInteger(text)) { + return *id; + } + return Error::OutOfRange; +} + +Result> RustHFTokenizer::encode( + const std::string& input, + int8_t bos, + int8_t eos) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (input.size() > static_cast(std::numeric_limits::max())) { + return Error::EncodeFailure; + } + + std::vector output(input.size()); + auto count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + static_cast(bos > 0 || eos > 0), + output.data(), + output.size()); + if (count < 0) { + return Error::EncodeFailure; + } + if (static_cast(count) > output.size()) { + output.resize(static_cast(count)); + count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + static_cast(bos > 0 || eos > 0), + output.data(), + output.size()); + if (count < 0 || static_cast(count) > output.size()) { + return Error::EncodeFailure; + } + } + output.resize(static_cast(count)); + return std::vector(output.begin(), output.end()); +} + +Result RustHFTokenizer::decode( + uint64_t /*prev_token*/, + uint64_t token, + bool skip_special_tokens) const { + if (!initialized_) { + return Error::Uninitialized; + } + std::string_view piece; + if (auto regular = token_map_->tryGetString(token)) { + piece = *regular; + } else if (auto added = added_token_map_->tryGetString(token)) { + if (skip_special_tokens && special_token_ids_.count(token) != 0) { + return std::string(); + } + piece = *added; + } else { + return Error::DecodeFailure; + } + + if (!byte_level_) { + return std::string(piece); + } + return decode_byte_level(piece); +} + +} // namespace tokenizers From 166923797174d706019ef10116fd2a457a964dcd Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 11:27:06 -0700 Subject: [PATCH 2/4] Fix Rust tokenizer adapter semantics Preserve independent BOS/EOS counts, avoid guessing a missing token ID, and reject .tok files whose decoder cannot yet be represented safely. Add C++ regression coverage and run the Rust checks in CI. AI-assisted-by: Codex --- CMakeLists.txt | 9 +- .../pytorch/tokenizers/rust_hf_tokenizer.h | 2 + rust_tokenizer/CMakeLists.txt | 28 +-- rust_tokenizer/README.md | 24 ++- rust_tokenizer/src/lib.rs | 23 +-- src/rust_hf_tokenizer.cpp | 102 ++++++---- test/CMakeLists.txt | 6 + test/test_rust_hf_tokenizer.cpp | 187 ++++++++++++++++++ 8 files changed, 291 insertions(+), 90 deletions(-) create mode 100644 test/test_rust_hf_tokenizer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f1eaa8fc..099eb477 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,11 +19,12 @@ project(Tokenizers) option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) -option( - TOKENIZERS_BUILD_HF_RUST_TOKENIZER - "Build the opt-in Hugging Face .tok tokenizer backend" OFF +option(TOKENIZERS_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face .tok tokenizer backend" OFF +) +option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" + OFF ) -option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" OFF) option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF ) diff --git a/include/pytorch/tokenizers/rust_hf_tokenizer.h b/include/pytorch/tokenizers/rust_hf_tokenizer.h index b75e8fc7..7699bb8b 100644 --- a/include/pytorch/tokenizers/rust_hf_tokenizer.h +++ b/include/pytorch/tokenizers/rust_hf_tokenizer.h @@ -50,6 +50,8 @@ class RustHFTokenizer final : public Tokenizer { std::optional token_map_; std::optional added_token_map_; std::unordered_set special_token_ids_; + bool has_bos_token_ = false; + bool has_eos_token_ = false; bool byte_level_ = false; }; diff --git a/rust_tokenizer/CMakeLists.txt b/rust_tokenizer/CMakeLists.txt index ee219656..a9bc4831 100644 --- a/rust_tokenizer/CMakeLists.txt +++ b/rust_tokenizer/CMakeLists.txt @@ -12,12 +12,8 @@ if(TOKENIZERS_OPTIMIZE_SIZE) endif() set(_cargo_target_dir ${CMAKE_CURRENT_BINARY_DIR}/cargo-target) -string( - CONCAT - _rust_library_name - ${CMAKE_STATIC_LIBRARY_PREFIX} - tokenizers_hf_ffi - ${CMAKE_STATIC_LIBRARY_SUFFIX} +string(CONCAT _rust_library_name ${CMAKE_STATIC_LIBRARY_PREFIX} + tokenizers_hf_ffi ${CMAKE_STATIC_LIBRARY_SUFFIX} ) set(_rust_library ${_cargo_target_dir}/${_rust_profile}/${_rust_library_name}) file(GLOB_RECURSE _rust_sources CONFIGURE_DEPENDS @@ -28,9 +24,9 @@ add_custom_command( OUTPUT ${_rust_library} COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_cargo_target_dir} - CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic - ${CARGO_EXECUTABLE} build --locked --manifest-path - ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile ${_rust_profile} + CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic ${CARGO_EXECUTABLE} build + --locked --manifest-path ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile + ${_rust_profile} DEPENDS ${_rust_sources} ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -51,23 +47,19 @@ set_target_properties( ) target_include_directories( tokenizers_hf_rust_tokenizer - PUBLIC - $ - $ + PUBLIC $ + $ ) target_compile_features(tokenizers_hf_rust_tokenizer PUBLIC cxx_std_17) -target_link_libraries( - tokenizers_hf_rust_tokenizer - PRIVATE tokenizers_hf_ffi -) +target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE tokenizers_hf_ffi) if(APPLE) target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE iconv) elseif(UNIX) find_package(Threads REQUIRED) target_link_libraries( - tokenizers_hf_rust_tokenizer - PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m rt util + tokenizers_hf_rust_tokenizer PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m rt + util ) endif() diff --git a/rust_tokenizer/README.md b/rust_tokenizer/README.md index 63f82a04..8c58d7a4 100644 --- a/rust_tokenizer/README.md +++ b/rust_tokenizer/README.md @@ -10,10 +10,10 @@ Use `-DTOKENIZERS_OPTIMIZE_SIZE=ON` for the Rust `minsize` profile; ExecuTorch forwards `EXECUTORCH_OPTIMIZE_SIZE` to this option. When enabled, the LLM runner uses Hugging Face's parser-free inference pipeline -for `.tok` files while retaining the existing ExecuTorch `Tokenizer` interface. -A directory input uses `/tokenizer.tok`. JSON tokenizers continue -through the existing C++ `HFTokenizer`; all other tokenizer fallbacks are -unchanged. +for byte-level `.tok` files while retaining the existing ExecuTorch `Tokenizer` +interface. A directory input uses `/tokenizer.tok`. JSON tokenizers +continue through the existing C++ `HFTokenizer`; all other tokenizer fallbacks +are unchanged. Create the artifact offline with the `tk-convert` tool from Hugging Face's [`feat/tok-format`](https://github.com/huggingface/tokenizers/tree/feat/tok-format) @@ -24,10 +24,15 @@ cargo run --release --manifest-path tokenizers/Cargo.toml -p tk-convert -- \ /path/to/tokenizer.json ``` -This writes `/path/to/tokenizer.tok`. The v1 container supports BPE, Unigram, -WordPiece, and WordLevel models, but intentionally supports only the -normalizers and pre-tokenizers represented by the format. Conversion fails -instead of silently dropping unsupported behavior. +This writes `/path/to/tokenizer.tok`. The v1 container's encoder supports BPE, +Unigram, WordPiece, and WordLevel models, but it does not yet serialize decoder +configuration. The ExecuTorch adapter therefore rejects non-byte-level files +instead of returning raw, incorrectly decoded vocabulary pieces. Conversion +fails instead of silently dropping unsupported normalizer or pre-tokenizer +behavior. + +The adapter also requires identifiable BOS and EOS IDs because the ExecuTorch +tokenizer interface cannot represent either value as absent. This experiment currently supports host CMake builds. Android, Apple framework, WASM, and Buck packaging still need explicit Rust target/toolchain integration. @@ -41,5 +46,4 @@ serde, training, and progress-bar code stay out of the runtime binary. The Rust pipeline owns encoding. The C++ compatibility layer reads vocabulary and special-token metadata from the same `.tok` image and applies byte-level decoding when the format marks the model as byte-level. Other decoder chains -are not represented by `.tok` v1 and therefore retain raw-piece incremental -decode behavior. +must wait until a future `.tok` format represents them. diff --git a/rust_tokenizer/src/lib.rs b/rust_tokenizer/src/lib.rs index 23c68431..acc3f1d2 100644 --- a/rust_tokenizer/src/lib.rs +++ b/rust_tokenizer/src/lib.rs @@ -52,7 +52,6 @@ pub unsafe extern "C" fn tokenizers_hf_encode( opaque: *const c_void, text: *const u8, text_len: usize, - add_special_tokens: u8, output: *mut u32, output_capacity: usize, ) -> isize { @@ -70,7 +69,9 @@ pub unsafe extern "C" fn tokenizers_hf_encode( return -1; }; - let Ok(tokens) = handle.tokenizer.encode(text, add_special_tokens != 0) else { + // The C++ Tokenizer interface controls BOS and EOS independently and + // permits repeated tokens, so the adapter applies them after this call. + let Ok(tokens) = handle.tokenizer.encode(text, false) else { return -1; }; let Ok(token_count) = isize::try_from(tokens.len()) else { @@ -360,29 +361,21 @@ mod tests { with_handle(|handle| { let text = b"hello"; let required = unsafe { - tokenizers_hf_encode( - handle, - text.as_ptr(), - text.len(), - 1, - std::ptr::null_mut(), - 0, - ) + tokenizers_hf_encode(handle, text.as_ptr(), text.len(), std::ptr::null_mut(), 0) }; - assert_eq!(required, 3); - let mut output = [0u32; 3]; + assert_eq!(required, 1); + let mut output = [0u32; 1]; let written = unsafe { tokenizers_hf_encode( handle, text.as_ptr(), text.len(), - 1, output.as_mut_ptr(), output.len(), ) }; - assert_eq!(written, 3); - assert_eq!(output, [2, 1, 2]); + assert_eq!(written, 1); + assert_eq!(output, [1]); }); } diff --git a/src/rust_hf_tokenizer.cpp b/src/rust_hf_tokenizer.cpp index 7604eeb2..92a3fd9b 100644 --- a/src/rust_hf_tokenizer.cpp +++ b/src/rust_hf_tokenizer.cpp @@ -23,7 +23,6 @@ intptr_t tokenizers_hf_encode( const void* handle, const uint8_t* text, size_t text_len, - uint8_t add_special_tokens, uint32_t* output, size_t output_capacity); intptr_t tokenizers_hf_token_count(const void* handle); @@ -35,10 +34,8 @@ int32_t tokenizers_hf_token_at( size_t* text_len, uint8_t* is_added, uint8_t* is_special); -int32_t tokenizers_hf_post_token( - const void* handle, - uint8_t suffix, - uint32_t* token); +int32_t +tokenizers_hf_post_token(const void* handle, uint8_t suffix, uint32_t* token); uint32_t tokenizers_hf_config_flags(const void* handle); void tokenizers_hf_destroy(void* handle); } @@ -50,6 +47,19 @@ namespace { // keeps these values stable as part of its on-disk schema. constexpr uint32_t kByteLevelFlag = 1U << 2; +bool is_bos_token(std::string_view token) { + return token == "" || token == "[CLS]" || token == "" || + token == "" || token == "<|bos|>" || token == "<|begin_of_text|>" || + token == "<|start_of_text|>" || token == "<|startoftext|>" || + token == "<|endoftext|>"; +} + +bool is_eos_token(std::string_view token) { + return token == "" || token == "[SEP]" || token == "" || + token == "" || token == "<|eos|>" || token == "<|end_of_text|>" || + token == "<|endoftext|>"; +} + std::optional byte_level_codepoint_to_byte(uint32_t codepoint) { if ((codepoint >= 33 && codepoint <= 126) || (codepoint >= 161 && codepoint <= 172) || @@ -95,8 +105,7 @@ std::string decode_byte_level(std::string_view piece) { return std::string(piece); } for (size_t offset = 1; offset < length; ++offset) { - const auto continuation = - static_cast(piece[index + offset]); + const auto continuation = static_cast(piece[index + offset]); if ((continuation & 0xC0) != 0x80) { return std::string(piece); } @@ -128,6 +137,8 @@ Error RustHFTokenizer::load(const std::string& path) { token_map_.reset(); added_token_map_.reset(); special_token_ids_.clear(); + has_bos_token_ = false; + has_eos_token_ = false; byte_level_ = false; vocab_size_ = 0; bos_tok_ = 0; @@ -148,16 +159,23 @@ Error RustHFTokenizer::load(const std::string& path) { if (!handle) { return Error::LoadFailure; } - const auto metadata_error = load_metadata(handle.get()); - if (metadata_error != Error::Ok) { - return metadata_error; - } const auto flags = tokenizers_hf_config_flags(handle.get()); if (flags == std::numeric_limits::max()) { return Error::ParseFailure; } byte_level_ = (flags & kByteLevelFlag) != 0; + if (!byte_level_) { + // The v1 .tok format does not serialize decoder configuration. Loading a + // non-byte-level tokenizer would therefore produce raw vocabulary pieces + // instead of decoded text. + return Error::LoadFailure; + } + + const auto metadata_error = load_metadata(handle.get()); + if (metadata_error != Error::Ok) { + return metadata_error; + } handle_ = std::move(handle); initialized_ = true; @@ -188,13 +206,8 @@ Error RustHFTokenizer::load_metadata(const void* handle) { uint8_t is_added = 0; uint8_t is_special = 0; if (tokenizers_hf_token_at( - handle, - index, - &id, - &text, - &text_len, - &is_added, - &is_special) != 0 || + handle, index, &id, &text, &text_len, &is_added, &is_special) != + 0 || (text == nullptr && text_len != 0)) { return Error::ParseFailure; } @@ -219,12 +232,10 @@ Error RustHFTokenizer::load_metadata(const void* handle) { added_tokens.reserve(added_ids.size()); for (auto& record : records) { if (record.special) { - if (record.text.find("bos") != std::string::npos || - record.text.find("begin") != std::string::npos) { + if (is_bos_token(record.text)) { bos_candidates.push_back(record.id); } - if (record.text.find("eos") != std::string::npos || - record.text.find("end") != std::string::npos) { + if (is_eos_token(record.text)) { eos_candidates.push_back(record.id); } } @@ -244,40 +255,36 @@ Error RustHFTokenizer::load_metadata(const void* handle) { return token_map.error(); } token_map_.emplace(std::move(*token_map)); - vocab_size_ = static_cast( - token_map_->size() + added_token_map_->size()); + vocab_size_ = + static_cast(token_map_->size() + added_token_map_->size()); uint32_t bos = 0; uint32_t eos = 0; - const auto bos_status = - tokenizers_hf_post_token(handle, 0, &bos); - const auto eos_status = - tokenizers_hf_post_token(handle, 1, &eos); + const auto bos_status = tokenizers_hf_post_token(handle, 0, &bos); + const auto eos_status = tokenizers_hf_post_token(handle, 1, &eos); if (bos_status < 0 || eos_status < 0) { return Error::ParseFailure; } if (bos_status == 0) { bos_tok_ = bos; + has_bos_token_ = true; } if (eos_status == 0) { eos_tok_ = eos; + has_eos_token_ = true; } - bool bos_found = bos_status == 0; - bool eos_found = eos_status == 0; - if (!bos_found || !eos_found) { - if (!bos_found && bos_candidates.size() == 1) { + if (!has_bos_token_ || !has_eos_token_) { + if (!has_bos_token_ && bos_candidates.size() == 1) { bos_tok_ = bos_candidates.front(); - bos_found = true; + has_bos_token_ = true; } - if (!eos_found && eos_candidates.size() == 1) { + if (!has_eos_token_ && eos_candidates.size() == 1) { eos_tok_ = eos_candidates.front(); - eos_found = true; + has_eos_token_ = true; } } - if (bos_found && !eos_found) { - eos_tok_ = bos_tok_; - } else if (!bos_found && eos_found) { - bos_tok_ = eos_tok_; + if (!has_bos_token_ || !has_eos_token_) { + return Error::ParseFailure; } return Error::Ok; } @@ -315,7 +322,11 @@ Result> RustHFTokenizer::encode( if (!initialized_) { return Error::Uninitialized; } - if (input.size() > static_cast(std::numeric_limits::max())) { + if (input.size() > + static_cast(std::numeric_limits::max())) { + return Error::EncodeFailure; + } + if ((bos > 0 && !has_bos_token_) || (eos > 0 && !has_eos_token_)) { return Error::EncodeFailure; } @@ -324,7 +335,6 @@ Result> RustHFTokenizer::encode( handle_.get(), reinterpret_cast(input.data()), input.size(), - static_cast(bos > 0 || eos > 0), output.data(), output.size()); if (count < 0) { @@ -336,7 +346,6 @@ Result> RustHFTokenizer::encode( handle_.get(), reinterpret_cast(input.data()), input.size(), - static_cast(bos > 0 || eos > 0), output.data(), output.size()); if (count < 0 || static_cast(count) > output.size()) { @@ -344,7 +353,14 @@ Result> RustHFTokenizer::encode( } } output.resize(static_cast(count)); - return std::vector(output.begin(), output.end()); + const auto bos_count = bos > 0 ? static_cast(bos) : 0; + const auto eos_count = eos > 0 ? static_cast(eos) : 0; + std::vector tokens; + tokens.reserve(output.size() + bos_count + eos_count); + tokens.insert(tokens.end(), bos_count, bos_tok_); + tokens.insert(tokens.end(), output.begin(), output.end()); + tokens.insert(tokens.end(), eos_count, eos_tok_); + return tokens; } Result RustHFTokenizer::decode( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b9e0a034..6e80b084 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,12 @@ foreach(test_source_file ${test_source_files}) get_filename_component(test_name ${test_source_file} NAME_WE) message(STATUS "Configuring unit test ${test_name}") add_executable(${test_name} ${test_source_file}) + if(test_name STREQUAL "test_rust_hf_tokenizer") + target_sources( + ${test_name} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src/rust_hf_tokenizer.cpp + ) + endif() target_include_directories( ${test_name} PRIVATE GTEST_INCLUDE_PATH ${TOKENIZERS_INCLUDE_DIRS} ) diff --git a/test/test_rust_hf_tokenizer.cpp b/test/test_rust_hf_tokenizer.cpp new file mode 100644 index 00000000..5a02b615 --- /dev/null +++ b/test/test_rust_hf_tokenizer.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +constexpr uint32_t kByteLevelFlag = 1U << 2; +uint32_t g_config_flags = kByteLevelFlag; +bool g_use_known_eos = true; + +struct TokenRecord { + uint32_t id; + std::string text; + bool added; + bool special; +}; + +const std::array kRecords = { + {{1, "", true, true}, + {2, "", true, true}, + {3, "hello", false, false}, + {4, "\xC4\xA0world", false, false}}}; + +class TemporaryTokFile { + public: + TemporaryTokFile() + : path_( + std::filesystem::temp_directory_path() / + "pytorch-tokenizers-rust-adapter-test.tok") { + std::ofstream(path_).put('\0'); + } + + ~TemporaryTokFile() { + std::error_code error; + std::filesystem::remove(path_, error); + } + + std::string string() const { + return path_.string(); + } + + private: + std::filesystem::path path_; +}; + +} // namespace + +extern "C" { + +void* tokenizers_hf_create(const char*) { + return new uint8_t(0); +} + +intptr_t tokenizers_hf_encode( + const void*, + const uint8_t*, + size_t, + uint32_t* output, + size_t output_capacity) { + constexpr std::array encoded = {3, 4}; + if (output_capacity < encoded.size()) { + return encoded.size(); + } + std::copy(encoded.begin(), encoded.end(), output); + return encoded.size(); +} + +intptr_t tokenizers_hf_token_count(const void*) { + return kRecords.size(); +} + +int32_t tokenizers_hf_token_at( + const void*, + size_t index, + uint32_t* id, + const uint8_t** text, + size_t* text_len, + uint8_t* is_added, + uint8_t* is_special) { + if (index >= kRecords.size()) { + return -1; + } + const auto& record = kRecords[index]; + static const std::string kUnknownEos = ""; + const auto& record_text = + index == 1 && !g_use_known_eos ? kUnknownEos : record.text; + *id = record.id; + *text = reinterpret_cast(record_text.data()); + *text_len = record_text.size(); + *is_added = record.added; + *is_special = record.special; + return 0; +} + +int32_t tokenizers_hf_post_token(const void*, uint8_t suffix, uint32_t* token) { + if (suffix != 0) { + return 1; + } + *token = 1; + return 0; +} + +uint32_t tokenizers_hf_config_flags(const void*) { + return g_config_flags; +} + +void tokenizers_hf_destroy(void* handle) { + delete static_cast(handle); +} + +} // extern "C" + +namespace tokenizers { +namespace { + +TEST(RustHFTokenizerTest, PreservesIndependentBosAndEosCounts) { + g_config_flags = kByteLevelFlag; + g_use_known_eos = true; + TemporaryTokFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + EXPECT_EQ(tokenizer.bos_tok(), 1); + EXPECT_EQ(tokenizer.eos_tok(), 2); + + auto tokens = tokenizer.encode("hello world", 2, 1); + ASSERT_TRUE(tokens.ok()); + EXPECT_EQ(*tokens, (std::vector{1, 1, 3, 4, 2})); + + auto bos_only_tokens = tokenizer.encode("hello world", 1, 0); + ASSERT_TRUE(bos_only_tokens.ok()); + EXPECT_EQ(*bos_only_tokens, (std::vector{1, 3, 4})); +} + +TEST(RustHFTokenizerTest, DecodesByteLevelPieces) { + g_config_flags = kByteLevelFlag; + g_use_known_eos = true; + TemporaryTokFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + + auto decoded = tokenizer.decode(3, 4); + ASSERT_TRUE(decoded.ok()); + EXPECT_EQ(*decoded, " world"); + + auto skipped = tokenizer.decode(4, 2, true); + ASSERT_TRUE(skipped.ok()); + EXPECT_TRUE(skipped->empty()); +} + +TEST(RustHFTokenizerTest, RejectsTokWithoutSupportedDecoder) { + g_use_known_eos = true; + TemporaryTokFile file; + RustHFTokenizer tokenizer; + g_config_flags = 0; + EXPECT_EQ(tokenizer.load(file.string()), Error::LoadFailure); + EXPECT_FALSE(tokenizer.is_loaded()); + g_config_flags = kByteLevelFlag; +} + +TEST(RustHFTokenizerTest, RejectsTokWithoutBosOrEosMetadata) { + TemporaryTokFile file; + RustHFTokenizer tokenizer; + g_config_flags = kByteLevelFlag; + g_use_known_eos = false; + EXPECT_EQ(tokenizer.load(file.string()), Error::ParseFailure); + EXPECT_FALSE(tokenizer.is_loaded()); + g_use_known_eos = true; +} + +} // namespace +} // namespace tokenizers From 5c68507a043ea2d56a0166109a4bb1a79d1a69fd Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 13:26:31 -0700 Subject: [PATCH 3/4] Support Hugging Face v1 JSON and experimental tok AI-assisted-by: Codex --- CMakeLists.txt | 9 +- .../pytorch/tokenizers/rust_hf_tokenizer.h | 1 + rust_tokenizer/CMakeLists.txt | 12 +- rust_tokenizer/Cargo.lock | 265 ++++-- rust_tokenizer/Cargo.toml | 42 +- rust_tokenizer/README.md | 78 +- rust_tokenizer/binary-size.md | 13 +- rust_tokenizer/src/lib.rs | 763 +++++++++++++----- src/rust_hf_tokenizer.cpp | 67 +- test/test_rust_hf_tokenizer.cpp | 46 ++ 10 files changed, 990 insertions(+), 306 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 099eb477..29c7ee40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,14 @@ option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) option(TOKENIZERS_BUILD_HF_RUST_TOKENIZER - "Build the opt-in Hugging Face .tok tokenizer backend" OFF + "Build the opt-in Hugging Face Rust tokenizer backend" OFF +) +set(TOKENIZERS_HF_RUST_FORMATS + "json" + CACHE STRING "Rust tokenizer formats: json, tok, or json,tok" +) +set_property( + CACHE TOKENIZERS_HF_RUST_FORMATS PROPERTY STRINGS json tok "json,tok" ) option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" OFF diff --git a/include/pytorch/tokenizers/rust_hf_tokenizer.h b/include/pytorch/tokenizers/rust_hf_tokenizer.h index 7699bb8b..61316c59 100644 --- a/include/pytorch/tokenizers/rust_hf_tokenizer.h +++ b/include/pytorch/tokenizers/rust_hf_tokenizer.h @@ -53,6 +53,7 @@ class RustHFTokenizer final : public Tokenizer { bool has_bos_token_ = false; bool has_eos_token_ = false; bool byte_level_ = false; + bool native_decoder_ = false; }; } // namespace tokenizers diff --git a/rust_tokenizer/CMakeLists.txt b/rust_tokenizer/CMakeLists.txt index a9bc4831..656ff9bc 100644 --- a/rust_tokenizer/CMakeLists.txt +++ b/rust_tokenizer/CMakeLists.txt @@ -6,6 +6,15 @@ find_program(CARGO_EXECUTABLE cargo REQUIRED) +if(NOT TOKENIZERS_HF_RUST_FORMATS STREQUAL "json" + AND NOT TOKENIZERS_HF_RUST_FORMATS STREQUAL "tok" + AND NOT TOKENIZERS_HF_RUST_FORMATS STREQUAL "json,tok" +) + message( + FATAL_ERROR "TOKENIZERS_HF_RUST_FORMATS must be json, tok, or json,tok" + ) +endif() + set(_rust_profile release) if(TOKENIZERS_OPTIMIZE_SIZE) set(_rust_profile minsize) @@ -25,7 +34,8 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_cargo_target_dir} CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic ${CARGO_EXECUTABLE} build - --locked --manifest-path ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile + --locked --no-default-features --features ${TOKENIZERS_HF_RUST_FORMATS} + --manifest-path ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile ${_rust_profile} DEPENDS ${_rust_sources} ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock diff --git a/rust_tokenizer/Cargo.lock b/rust_tokenizer/Cargo.lock index 87eca543..0f01a9e7 100644 --- a/rust_tokenizer/Cargo.lock +++ b/rust_tokenizer/Cargo.lock @@ -68,6 +68,42 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitcannon" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfc3924e96bdcf270ede3ff8e0713b68edaf2daa3ed1661467ed58cc5630fba" +dependencies = [ + "ahash", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -102,6 +138,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" @@ -110,12 +152,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cacheline-ef" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af737c6c59cb018ecbe6472cbdf86d39c59d78252febfe311953a991b6e4ed85" +checksum = "6e701af8ea9f58c4ba18793a5d5dffd791cda99bf766976c59c583939d180fd5" dependencies = [ "common_traits", - "mem_dbg 0.3.4", + "mem_dbg", ] [[package]] @@ -232,9 +274,9 @@ dependencies = [ [[package]] name = "common_traits" -version = "0.11.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda9ae1f26adcae83adb2e92f69cf59421f2a277a942f49f8e59f2fcbd7cf062" +checksum = "d8e0d7d0c5977b1e87f759d406740c38227125f374f54d81bc197e877f2c5f26" dependencies = [ "anyhow", "half", @@ -432,11 +474,14 @@ dependencies = [ ] [[package]] -name = "pytorch-tokenizers-hf-ffi" -version = "0.1.0" +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" dependencies = [ - "tk-encode", - "tk-serialization", + "bit-set", + "regex-automata", + "regex-syntax", ] [[package]] @@ -554,6 +599,12 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +[[package]] +name = "hifijson" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "242402749acf71e6f32f5857598b7002c4058a4e3c3b22b4c7d51cab9aea754e" + [[package]] name = "ident_case" version = "1.0.1" @@ -562,23 +613,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "impl-tools" -version = "0.10.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" +checksum = "06c8567c75b167cdda5ecee024da9cc9581d41bd52460e2e0e768967972aeef1" dependencies = [ "autocfg", "impl-tools-lib", - "proc-macro-error2", + "proc-macro-error3", "syn 2.0.119", ] [[package]] name = "impl-tools-lib" -version = "0.11.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" +checksum = "c81eb99ba7c6145aeccb8857ddc0c611cd52440091dcfddd605b6a930c9afbf4" dependencies = [ - "proc-macro-error2", + "proc-macro-error3", "proc-macro2", "quote", "syn 2.0.119", @@ -673,16 +724,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" -[[package]] -name = "mem_dbg" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728cc9dc97593cd22f7bc81fbef70a2d391d7a9a855e7d658b653318124a6cf0" -dependencies = [ - "bitflags", - "mem_dbg-derive 0.2.1", -] - [[package]] name = "mem_dbg" version = "0.4.4" @@ -691,18 +732,7 @@ checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" dependencies = [ "bitflags", "hashbrown", - "mem_dbg-derive 0.3.4", -] - -[[package]] -name = "mem_dbg-derive" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84f40c93b0508d5565db79a814d02d5b2545967205ce44be211592aafa34d6c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "mem_dbg-derive", ] [[package]] @@ -722,6 +752,12 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "monostate" version = "0.1.18" @@ -750,6 +786,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -839,22 +885,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" [[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "proc-macro-error-attr3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" dependencies = [ "proc-macro2", "quote", ] [[package]] -name = "proc-macro-error2" -version = "2.0.1" +name = "proc-macro-error3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" dependencies = [ - "proc-macro-error-attr2", + "proc-macro-error-attr3", "proc-macro2", "quote", ] @@ -870,8 +916,9 @@ dependencies = [ [[package]] name = "ptr_hash" -version = "2.0.2" -source = "git+https://github.com/ArthurZucker/PtrHash?rev=fff63a67eec9b48b693b0d3d2b253db78c0b9858#fff63a67eec9b48b693b0d3d2b253db78c0b9858" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d217d8cc2d882b70ba0602224621e35039c7f4c09f158ba8a560be9cd48ff7" dependencies = [ "bitvec", "cacheline-ef", @@ -880,16 +927,29 @@ dependencies = [ "fxhash", "itertools 0.15.0", "log", - "mem_dbg 0.4.4", + "mem_dbg", "prefetch-index", "rand 0.10.2", "rand_chacha 0.10.0", + "rayon", "rdst", "sucds", "tempfile", "xxhash-rust", ] +[[package]] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +dependencies = [ + "serde_json", + "tk-convert", + "tk-encode 0.1.0-rc.2", + "tk-encode 0.23.2-dev.0", + "tk-serialization", + "tk-serialize", +] + [[package]] name = "quote" version = "1.0.47" @@ -1003,6 +1063,7 @@ dependencies = [ "block-pseudorand", "criterion", "partition", + "rayon", "tikv-jemallocator", "voracious_radix_sort", ] @@ -1061,6 +1122,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1125,6 +1195,24 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1238,6 +1326,49 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tk-convert" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae9aa847b762e874fe27be83b120f63fc76fd7f555d254540d16e044660dfeb" +dependencies = [ + "bitcannon", + "serde_json", + "thiserror", +] + +[[package]] +name = "tk-encode" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b28a17e23359d88085a838d0c893875f7d220262368eeafdfd662577b1ba9ee" +dependencies = [ + "ahash", + "bitcannon", + "dary_heap", + "fancy-regex", + "itertools 0.15.0", + "log", + "memchr", + "paste", + "ptr_hash", + "rand 0.10.2", + "spm_precompiled", + "thiserror", + "unicode-normalization", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", + "wide", + "yada", +] + [[package]] name = "tk-encode" version = "0.23.2-dev.0" @@ -1271,12 +1402,42 @@ name = "tk-serialization" version = "0.23.2-dev.0" source = "git+https://github.com/huggingface/tokenizers?rev=054bdf469b2cf416c0da953923d88c82f4d765b5#054bdf469b2cf416c0da953923d88c82f4d765b5" +[[package]] +name = "tk-serialize" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39d1d911331f6c6e6a626bb4a67b3ddc86bceab6a5ee50918b1009bbda668fe" +dependencies = [ + "base64 0.22.1", + "bitcannon", + "hifijson", + "tk-encode 0.1.0-rc.2", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -1378,6 +1539,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wide" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d920ac99c3c8edce110cb8d07dbb324d6d026011dce85b1e9355b70f0adacc4f" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi-util" version = "0.1.11" diff --git a/rust_tokenizer/Cargo.toml b/rust_tokenizer/Cargo.toml index 167d7de8..cbc58fa6 100644 --- a/rust_tokenizer/Cargo.toml +++ b/rust_tokenizer/Cargo.toml @@ -8,19 +8,43 @@ publish = false name = "tokenizers_hf_ffi" crate-type = ["staticlib"] +[features] +default = ["json"] +json = [ + "dep:serde_json", + "dep:tk-convert-json", + "dep:tk-encode-json", + "dep:tk-serialize-json", +] +tok = ["dep:tk-encode-tok", "dep:tk-serialization"] + [dependencies] -tk-encode = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", default-features = false } -tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5" } +serde_json = { version = "1.0", optional = true } +tk-convert-json = { package = "tk-convert", version = "=0.1.0-rc.2", optional = true, default-features = false } +tk-encode-json = { package = "tk-encode", version = "=0.1.0-rc.2", optional = true, default-features = false, features = [ + "bpe", + "fancy-regex", + "normalizers", + "unicode-scripts", + "unigram", + "wordlevel", + "wordpiece", +] } +tk-serialize-json = { package = "tk-serialize", version = "=0.1.0-rc.2", optional = true, default-features = false, features = [ + "bpe", + "deserialize", + "fancy-regex", + "normalizers", + "unicode-scripts", + "unigram", + "wordlevel", + "wordpiece", +] } +tk-encode-tok = { package = "tk-encode", git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", optional = true, default-features = false } +tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", optional = true } [dev-dependencies] tk-serialization = { git = "https://github.com/huggingface/tokenizers", rev = "054bdf469b2cf416c0da953923d88c82f4d765b5", features = ["write"] } - -# The `.tok` branch relies on PtrHash's dependency gates until they are -# available in a crates.io release. Pin the same revision rather than following -# the mutable upstream branch. -[patch.crates-io] -ptr_hash = { git = "https://github.com/ArthurZucker/PtrHash", rev = "fff63a67eec9b48b693b0d3d2b253db78c0b9858" } - [profile.release] codegen-units = 1 lto = "fat" diff --git a/rust_tokenizer/README.md b/rust_tokenizer/README.md index 8c58d7a4..2f844607 100644 --- a/rust_tokenizer/README.md +++ b/rust_tokenizer/README.md @@ -1,21 +1,43 @@ -# Experimental Hugging Face Rust tokenizer backend - -This backend is opt-in and disabled by default. Enable it with -`-DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON`. When disabled, CMake does not enter -this directory, invoke Cargo, build the Rust archive, or compile the C++ -bridge, so the default tokenizer library is unchanged. ExecuTorch users can -enable the same target with both `-DEXECUTORCH_BUILD_EXTENSION_LLM=ON` and -`-DEXECUTORCH_BUILD_HF_RUST_TOKENIZER=ON`. +# Hugging Face Rust tokenizer backend + +This backend is opt-in and disabled by default. It can load the standard +Hugging Face `tokenizer.json` format through the v1 release candidate, the +experimental `.tok` format, or both. Enable it with +`-DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON`. When disabled, CMake does not invoke +Cargo or compile the C++ bridge, so the default tokenizer library is unchanged. +ExecuTorch users also set `-DEXECUTORCH_BUILD_EXTENSION_LLM=ON`. + +Select the linked formats with `TOKENIZERS_HF_RUST_FORMATS` (or +`EXECUTORCH_HF_RUST_FORMATS` from the ExecuTorch root): + +| Value | Runtime input | Dependency source | +|---|---|---| +| `json` | `tokenizer.json` | published Hugging Face v1 RC crates | +| `tok` | `tokenizer.tok` | pinned experimental `.tok` branch | +| `json,tok` | both; JSON is preferred for directory inputs | both stacks | + +The default is `json`, keeping the supported RC path independent of the +experimental branch. A size-focused `.tok`-only build is: + +```sh +cmake -S . -B build \ + -DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON \ + -DTOKENIZERS_HF_RUST_FORMATS=tok \ + -DTOKENIZERS_OPTIMIZE_SIZE=ON +``` + Use `-DTOKENIZERS_OPTIMIZE_SIZE=ON` for the Rust `minsize` profile; ExecuTorch forwards `EXECUTORCH_OPTIMIZE_SIZE` to this option. -When enabled, the LLM runner uses Hugging Face's parser-free inference pipeline -for byte-level `.tok` files while retaining the existing ExecuTorch `Tokenizer` -interface. A directory input uses `/tokenizer.tok`. JSON tokenizers -continue through the existing C++ `HFTokenizer`; all other tokenizer fallbacks -are unchanged. +The JSON path uses Hugging Face `tokenizers` v1.0.0-rc.2's `tk-encode`, +`tk-serialize`, and `tk-convert` crates. It accepts normal Hub +`tokenizer.json` files, canonicalizes legacy v1 JSON in memory, and uses the +upstream decoder chain. If a directory is passed, it reads `tokenizer.json` and +uses sibling `tokenizer_config.json` or `special_tokens_map.json` for BOS/EOS +metadata when available. -Create the artifact offline with the `tk-convert` tool from Hugging Face's +The `.tok` path stays available independently. Create its artifact offline with +the `tk-convert` tool from Hugging Face's [`feat/tok-format`](https://github.com/huggingface/tokenizers/tree/feat/tok-format) branch: @@ -34,16 +56,22 @@ behavior. The adapter also requires identifiable BOS and EOS IDs because the ExecuTorch tokenizer interface cannot represent either value as absent. -This experiment currently supports host CMake builds. Android, Apple framework, -WASM, and Buck packaging still need explicit Rust target/toolchain integration. -The build requires Cargo and fetches the pinned Rust dependencies on its first -run. +An explicit file path always selects its format. For a directory containing +both files, JSON is preferred. If the selected Rust backend cannot load a JSON +file, the LLM runner retains its existing C++ `HFTokenizer` fallback. + +This integration currently supports host CMake builds. Android, Apple +framework, WASM, and Buck packaging still need explicit Rust target/toolchain +integration. The build requires Cargo and fetches its pinned dependencies on +the first run. The Rust dependencies are pinned to Hugging Face tokenizers commit -`054bdf469b2cf416c0da953923d88c82f4d765b5` from `feat/tok-format`. Only -`tk-encode` and the zero-dependency `.tok` reader are linked; JSON conversion, -serde, training, and progress-bar code stay out of the runtime binary. The -Rust pipeline owns encoding. The C++ compatibility layer reads vocabulary and -special-token metadata from the same `.tok` image and applies byte-level -decoding when the format marks the model as byte-level. Other decoder chains -must wait until a future `.tok` format represents them. +`054bdf469b2cf416c0da953923d88c82f4d765b5` from `feat/tok-format`. A `tok` +build links only that pipeline and reader: the released JSON reader, +canonicalizer, serde, training, and progress-bar code stay out of the binary. +The `.tok` format still lacks decoder configuration, so only byte-level files +are accepted. The JSON path does not have that limitation. + +Hugging Face has announced inference-only C and C++ bindings for ExecuTorch and +llama.cpp on the v1 roadmap. The local C ABI is deliberately small so the JSON +implementation can move to that upstream binding when it stabilizes. diff --git a/rust_tokenizer/binary-size.md b/rust_tokenizer/binary-size.md index 8ad6ab6e..da9dea8d 100644 --- a/rust_tokenizer/binary-size.md +++ b/rust_tokenizer/binary-size.md @@ -1,4 +1,4 @@ -# Hugging Face `.tok` backend binary size +# Hugging Face Rust tokenizer backend binary size Measured on Apple arm64 with Rust 1.98.1, `CMAKE_BUILD_TYPE=Release`, `TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. The smoke binary @@ -8,7 +8,12 @@ decodes both output tokens. | Configuration | Stripped | Gzipped | |---|---:|---:| | Default (`TOKENIZERS_BUILD_HF_RUST_TOKENIZER=OFF`) | 0 B added | 0 B added | -| Opt-in `.tok` backend | 591,040 B | 298,961 B | +| `.tok` only | 817,888 B | 377,292 B | +| JSON only | 1,921,824 B | 902,655 B | +| JSON and `.tok` | 2,105,024 B | 1,007,448 B | -The OFF configuration exposes no Rust CMake target and produces no Cargo build -directory. +The `.tok` row uses `TOKENIZERS_HF_RUST_FORMATS=tok`; it does not link the JSON +reader or canonicalizer. The JSON row uses the published v1.0.0-rc.2 crates. +These are complete smoke executables that load and round-trip GPT-2, not archive +sizes. The OFF configuration exposes no Rust CMake target and produces no Cargo +build directory. diff --git a/rust_tokenizer/src/lib.rs b/rust_tokenizer/src/lib.rs index acc3f1d2..93af7fbc 100644 --- a/rust_tokenizer/src/lib.rs +++ b/rust_tokenizer/src/lib.rs @@ -1,24 +1,364 @@ use std::ffi::{CStr, c_char, c_void}; use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; -use tk_encode::pipeline::PipelineTokenizer; +#[cfg(feature = "json")] +use serde_json::Value; +#[cfg(feature = "json")] +use tk_encode_json::pipeline::{ + EncodeOptions, Override, PipelineModel, PipelineTokenizer as JsonTokenizer, +}; +#[cfg(feature = "tok")] +use tk_encode_tok::pipeline::PipelineTokenizer as TokTokenizer; +#[cfg(feature = "tok")] use tk_serialization::{AddedEntry, Entry, TokFile, added_flag, kind}; +#[cfg(not(any(feature = "json", feature = "tok")))] +compile_error!("enable at least one of the `json` or `tok` features"); + +#[cfg(feature = "json")] +const BYTE_LEVEL_FLAG: u32 = 1 << 2; + +struct TokenRecord { + id: u32, + text: String, + added: bool, + special: bool, +} + +struct Metadata { + records: Vec, + bos: Option, + eos: Option, + flags: u32, + native_decoder: bool, +} + +enum Backend { + #[cfg(feature = "json")] + Json(JsonTokenizer), + #[cfg(feature = "tok")] + Tok { + tokenizer: Box, + _file: TokFile, + }, +} + struct Handle { - tokenizer: PipelineTokenizer, - // Keep the aligned `.tok` image alive. `PipelineTokenizer::from_tok` - // currently owns the structures it builds, but retaining the image makes - // the FFI safe if the upstream reader starts borrowing sections later. - file: TokFile, + backend: Backend, + metadata: Metadata, +} + +impl Handle { + fn load(path: &Path) -> Result { + match path.extension().and_then(|extension| extension.to_str()) { + #[cfg(feature = "json")] + Some("json") => Self::load_json(path), + #[cfg(feature = "tok")] + Some("tok") => Self::load_tok(path), + _ => Err(()), + } + } + + #[cfg(feature = "json")] + fn load_json(path: &Path) -> Result { + let canonical = tk_convert_json::canonicalize_file(path).map_err(|_| ())?; + let tokenizer = tk_serialize_json::from_json(&canonical).map_err(|_| ())?; + let mut metadata = json_metadata(&canonical)?; + + let post_processor = tokenizer.get_post_processor(); + let post_bos = post_processor + .single + .prefix + .first() + .map(|(token, _)| token.id()); + let post_eos = post_processor + .single + .suffix + .last() + .map(|(token, _)| token.id()); + let role_bos = tokenizer + .get_token_for_role("bos_token") + .and_then(|token| token_id(&metadata.records, token)); + let role_eos = tokenizer + .get_token_for_role("eos_token") + .and_then(|token| token_id(&metadata.records, token)); + let sidecar = sidecar_special_tokens(path); + metadata.bos = role_bos + .or_else(|| { + sidecar + .0 + .and_then(|token| token_id(&metadata.records, &token)) + }) + .or(post_bos); + metadata.eos = role_eos + .or_else(|| { + sidecar + .1 + .and_then(|token| token_id(&metadata.records, &token)) + }) + .or(post_eos); + if matches!( + tokenizer.get_model(), + PipelineModel::BPE(model) if model.is_byte_level() + ) { + metadata.flags |= BYTE_LEVEL_FLAG; + } + Ok(Self { + backend: Backend::Json(tokenizer), + metadata, + }) + } + + #[cfg(feature = "tok")] + fn load_tok(path: &Path) -> Result { + let file = TokFile::open(path).map_err(|_| ())?; + let tokenizer = TokTokenizer::from_tok(file.bytes()).map_err(|_| ())?; + let reader = file.reader().map_err(|_| ())?; + let vocab = reader.require::(kind::VOCAB_ENTRY).map_err(|_| ())?; + let vocab_slab = reader.require::(kind::VOCAB_SLAB).map_err(|_| ())?; + let added = reader + .section::(kind::ADDED_ENTRY) + .map_err(|_| ())?; + let added_slab = reader.section::(kind::ADDED_SLAB).map_err(|_| ())?; + let mut records = Vec::with_capacity(vocab.len() + added.len()); + for entry in vocab { + records.push(TokenRecord { + id: entry.id, + text: slab_string(vocab_slab, entry.start, entry.len)?, + added: false, + special: false, + }); + } + for entry in added { + records.push(TokenRecord { + id: entry.id, + text: slab_string(added_slab, entry.start, entry.len)?, + added: true, + special: entry.flags & added_flag::SPECIAL != 0, + }); + } + let prefix = reader + .section::(kind::POST_PREFIX) + .map_err(|_| ())? + .first() + .copied(); + let suffix = reader + .section::(kind::POST_SUFFIX) + .map_err(|_| ())? + .last() + .copied(); + let flags = reader.config.flags; + Ok(Self { + backend: Backend::Tok { + tokenizer: Box::new(tokenizer), + _file: file, + }, + metadata: Metadata { + records, + bos: prefix, + eos: suffix, + flags, + native_decoder: false, + }, + }) + } + + fn encode(&self, text: &str) -> Result, ()> { + match &self.backend { + #[cfg(feature = "json")] + Backend::Json(tokenizer) => { + let options = EncodeOptions { + padding: Override::Off, + truncation: Override::Off, + ..EncodeOptions::no_specials() + }; + let mut encodings = tokenizer.encode(text, &options).wait().map_err(|_| ())?; + if encodings.len() != 1 { + return Err(()); + } + Ok(encodings + .pop() + .expect("one encoding") + .ids() + .iter() + .map(|token| token.id()) + .collect()) + } + #[cfg(feature = "tok")] + Backend::Tok { tokenizer, .. } => tokenizer + .encode(text, false) + .map(|tokens| tokens.into_iter().map(|token| token.id).collect()) + .map_err(|_| ()), + } + } + + fn decode(&self, previous: Option, token: u32, skip_special: bool) -> Result { + #[cfg(not(feature = "json"))] + let _ = (previous, token, skip_special); + match &self.backend { + #[cfg(feature = "json")] + Backend::Json(tokenizer) => { + let current = tokenizer.decode(&[token], skip_special).map_err(|_| ())?; + let Some(previous) = previous else { + return Ok(current); + }; + let prefix = tokenizer + .decode(&[previous], skip_special) + .map_err(|_| ())?; + let pair = tokenizer + .decode(&[previous, token], skip_special) + .map_err(|_| ())?; + Ok(pair.strip_prefix(&prefix).unwrap_or(¤t).to_string()) + } + #[cfg(feature = "tok")] + Backend::Tok { .. } => Err(()), + } + } +} + +#[cfg(feature = "tok")] +fn slab_string(slab: &[u8], start: u32, len: u32) -> Result { + let start = start as usize; + let end = start.checked_add(len as usize).ok_or(())?; + std::str::from_utf8(slab.get(start..end).ok_or(())?) + .map(str::to_owned) + .map_err(|_| ()) +} + +#[cfg(feature = "json")] +fn token_id(records: &[TokenRecord], text: &str) -> Option { + records + .iter() + .find(|record| record.text == text) + .map(|record| record.id) +} + +#[cfg(feature = "json")] +fn json_metadata(text: &str) -> Result { + let document: Value = serde_json::from_str(text).map_err(|_| ())?; + let model = document.get("model").and_then(Value::as_object).ok_or(())?; + let vocab = model.get("vocab").ok_or(())?; + let mut records = Vec::new(); + if let Some(entries) = vocab.as_object() { + for (token, id) in entries { + records.push(TokenRecord { + id: value_u32(id)?, + text: token.clone(), + added: false, + special: false, + }); + } + } else if let Some(entries) = vocab.as_array() { + for (id, entry) in entries.iter().enumerate() { + let token = entry + .as_array() + .and_then(|fields| fields.first()) + .and_then(Value::as_str) + .ok_or(())?; + records.push(TokenRecord { + id: u32::try_from(id).map_err(|_| ())?, + text: token.to_owned(), + added: false, + special: false, + }); + } + } else { + return Err(()); + } + + if let Some(added_tokens) = document.get("added_tokens").and_then(Value::as_array) { + for token in added_tokens { + records.push(TokenRecord { + id: value_u32(token.get("id").ok_or(())?)?, + text: token + .get("content") + .and_then(Value::as_str) + .ok_or(())? + .to_owned(), + added: true, + special: token + .get("special") + .and_then(Value::as_bool) + .unwrap_or(false), + }); + } + } + + let roles = document.get("role_to_token").and_then(Value::as_object); + let bos = roles + .and_then(|roles| roles.get("bos_token")) + .and_then(special_token_text) + .and_then(|token| token_id(&records, token)); + let eos = roles + .and_then(|roles| roles.get("eos_token")) + .and_then(special_token_text) + .and_then(|token| token_id(&records, token)); + Ok(Metadata { + records, + bos, + eos, + flags: 0, + native_decoder: true, + }) +} + +#[cfg(feature = "json")] +fn value_u32(value: &Value) -> Result { + value + .as_u64() + .and_then(|id| u32::try_from(id).ok()) + .ok_or(()) +} + +#[cfg(feature = "json")] +fn special_token_text(value: &Value) -> Option<&str> { + value + .as_str() + .or_else(|| value.get("content").and_then(Value::as_str)) } -/// Create a tokenizer from a `.tok` path. +#[cfg(feature = "json")] +fn sidecar_special_tokens(path: &Path) -> (Option, Option) { + let mut bos = None; + let mut eos = None; + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + for filename in ["tokenizer_config.json", "special_tokens_map.json"] { + let Ok(text) = std::fs::read_to_string(parent.join(filename)) else { + continue; + }; + let Ok(document) = serde_json::from_str::(&text) else { + continue; + }; + if bos.is_none() { + bos = document + .get("bos_token") + .and_then(special_token_text) + .map(str::to_owned); + } + if eos.is_none() { + eos = document + .get("eos_token") + .and_then(special_token_text) + .map(str::to_owned); + } + } + (bos, eos) +} + +fn candidate_paths(path: &Path) -> Vec { + if !path.is_dir() { + return vec![path.to_owned()]; + } + vec![path.join("tokenizer.json"), path.join("tokenizer.tok")] +} + +/// Create a tokenizer from a `tokenizer.json`, `.tok`, or directory containing one. /// /// # Safety /// -/// `path` must be null or point to a valid NUL-terminated C string for the -/// duration of this call. A non-null return value must eventually be passed -/// exactly once to [`tokenizers_hf_destroy`]. +/// `path` must be null or point to a valid NUL-terminated C string for the duration of this call. +/// A non-null return value must eventually be passed exactly once to [`tokenizers_hf_destroy`]. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_void { catch_unwind(AssertUnwindSafe(|| { @@ -28,13 +368,12 @@ pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_voi let Ok(path) = unsafe { CStr::from_ptr(path) }.to_str() else { return std::ptr::null_mut(); }; - let Ok(file) = TokFile::open(path) else { - return std::ptr::null_mut(); - }; - let Ok(tokenizer) = PipelineTokenizer::from_tok(file.bytes()) else { - return std::ptr::null_mut(); - }; - Box::into_raw(Box::new(Handle { tokenizer, file })).cast() + for candidate in candidate_paths(Path::new(path)) { + if let Ok(handle) = Handle::load(&candidate) { + return Box::into_raw(Box::new(handle)).cast(); + } + } + std::ptr::null_mut() })) .unwrap_or(std::ptr::null_mut()) } @@ -43,9 +382,8 @@ pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_voi /// /// # Safety /// -/// `opaque` must be a live handle returned by -/// [`tokenizers_hf_create`]. `text` must address `text_len` readable -/// bytes unless `text_len` is zero. `output` must address `output_capacity` +/// `opaque` must be a live handle returned by [`tokenizers_hf_create`]. `text` must address +/// `text_len` readable bytes unless the length is zero. `output` must address `output_capacity` /// writable `u32` values unless the capacity is zero. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_encode( @@ -59,7 +397,6 @@ pub unsafe extern "C" fn tokenizers_hf_encode( if opaque.is_null() || (text.is_null() && text_len != 0) { return -1; } - let handle = unsafe { &*opaque.cast::() }; let bytes = if text_len == 0 { &[] } else { @@ -68,10 +405,7 @@ pub unsafe extern "C" fn tokenizers_hf_encode( let Ok(text) = std::str::from_utf8(bytes) else { return -1; }; - - // The C++ Tokenizer interface controls BOS and EOS independently and - // permits repeated tokens, so the adapter applies them after this call. - let Ok(tokens) = handle.tokenizer.encode(text, false) else { + let Ok(tokens) = (unsafe { &*opaque.cast::() }).encode(text) else { return -1; }; let Ok(token_count) = isize::try_from(tokens.len()) else { @@ -83,57 +417,90 @@ pub unsafe extern "C" fn tokenizers_hf_encode( if !tokens.is_empty() && output.is_null() { return -1; } - for (index, token) in tokens.iter().enumerate() { - unsafe { output.add(index).write(token.id) }; + if !tokens.is_empty() { + unsafe { std::ptr::copy_nonoverlapping(tokens.as_ptr(), output, tokens.len()) }; } token_count })) .unwrap_or(-1) } -/// Return the number of vocabulary and added-token records in the `.tok`. +/// Decode one token with optional previous-token context into caller-owned byte storage. /// -/// This is a record count rather than a vocabulary size because an added token -/// can deliberately reuse a model-vocabulary id. The C++ compatibility layer -/// removes those duplicates while constructing its lookup maps. +/// `has_previous == 0` ignores `previous`. The return value is the required byte count, or -1 +/// when this backend has no serialized decoder (the `.tok` format) or decoding fails. /// /// # Safety /// -/// `opaque` must be null or a live handle returned by -/// [`tokenizers_hf_create`]. +/// `opaque` must be a live handle. `output` must address `output_capacity` writable bytes unless +/// the capacity is zero. #[unsafe(no_mangle)] -pub unsafe extern "C" fn tokenizers_hf_token_count(opaque: *const c_void) -> isize { +pub unsafe extern "C" fn tokenizers_hf_decode( + opaque: *const c_void, + previous: u32, + has_previous: u8, + token: u32, + skip_special: u8, + output: *mut u8, + output_capacity: usize, +) -> isize { catch_unwind(AssertUnwindSafe(|| { if opaque.is_null() { return -1; } - let handle = unsafe { &*opaque.cast::() }; - let Ok(reader) = handle.file.reader() else { + let previous = (has_previous != 0).then_some(previous); + let Ok(decoded) = + (unsafe { &*opaque.cast::() }).decode(previous, token, skip_special != 0) + else { return -1; }; - let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { + let Ok(byte_count) = isize::try_from(decoded.len()) else { return -1; }; - let Ok(added) = reader.section::(kind::ADDED_ENTRY) else { + if decoded.len() > output_capacity { + return byte_count; + } + if !decoded.is_empty() && output.is_null() { return -1; - }; - vocab - .len() - .checked_add(added.len()) - .and_then(|count| isize::try_from(count).ok()) - .unwrap_or(-1) + } + if !decoded.is_empty() { + unsafe { + std::ptr::copy_nonoverlapping(decoded.as_ptr(), output, decoded.len()); + } + } + byte_count + })) + .unwrap_or(-1) +} + +/// Return the number of vocabulary and added-token records. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_count(opaque: *const c_void) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + isize::try_from( + (unsafe { &*opaque.cast::() }) + .metadata + .records + .len(), + ) + .unwrap_or(-1) })) .unwrap_or(-1) } -/// Read one vocabulary record. Returned string bytes borrow `opaque` and stay -/// valid until the handle is destroyed. +/// Read one vocabulary record. Returned string bytes borrow `opaque` and remain valid until the +/// handle is destroyed. /// /// # Safety /// -/// `opaque` must be a live handle returned by -/// [`tokenizers_hf_create`]. Every output argument must point to -/// writable storage of its declared type. +/// `opaque` must be a live handle. Every output argument must point to writable storage. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_token_at( opaque: *const c_void, @@ -155,67 +522,26 @@ pub unsafe extern "C" fn tokenizers_hf_token_at( return -1; } let handle = unsafe { &*opaque.cast::() }; - let Ok(reader) = handle.file.reader() else { - return -1; - }; - let Ok(vocab) = reader.require::(kind::VOCAB_ENTRY) else { - return -1; - }; - let (slab, start, len, token_id, added, special) = if index < vocab.len() { - let Ok(slab) = reader.require::(kind::VOCAB_SLAB) else { - return -1; - }; - let entry = vocab[index]; - (slab, entry.start, entry.len, entry.id, false, false) - } else { - let Ok(entries) = reader.section::(kind::ADDED_ENTRY) else { - return -1; - }; - let Some(entry) = entries.get(index - vocab.len()).copied() else { - return -1; - }; - let Ok(slab) = reader.section::(kind::ADDED_SLAB) else { - return -1; - }; - ( - slab, - entry.start, - entry.len, - entry.id, - true, - entry.flags & added_flag::SPECIAL != 0, - ) - }; - let Some(end) = (start as usize).checked_add(len as usize) else { - return -1; - }; - let Some(bytes) = slab.get(start as usize..end) else { + let Some(record) = handle.metadata.records.get(index) else { return -1; }; - if std::str::from_utf8(bytes).is_err() { - return -1; - } unsafe { - id.write(token_id); - text.write(bytes.as_ptr()); - text_len.write(bytes.len()); - is_added.write(u8::from(added)); - is_special.write(u8::from(special)); + id.write(record.id); + text.write(record.text.as_ptr()); + text_len.write(record.text.len()); + is_added.write(u8::from(record.added)); + is_special.write(u8::from(record.special)); } 0 })) .unwrap_or(-1) } -/// Return the first prefix token (`suffix == 0`) or last suffix token -/// (`suffix != 0`) from the `.tok` post-processor. Returns 0 when present, 1 -/// when the requested side is empty, and -1 on error. +/// Return the configured BOS (`suffix == 0`) or EOS (`suffix != 0`) token. /// /// # Safety /// -/// `opaque` must be a live handle returned by -/// [`tokenizers_hf_create`] and `token` must point to a writable -/// `u32`. +/// `opaque` must be a live handle and `token` must point to writable storage. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_post_token( opaque: *const c_void, @@ -226,59 +552,63 @@ pub unsafe extern "C" fn tokenizers_hf_post_token( if opaque.is_null() || token.is_null() { return -1; } - let handle = unsafe { &*opaque.cast::() }; - let Ok(reader) = handle.file.reader() else { - return -1; - }; - let Ok(tokens) = reader.section::(if suffix == 0 { - kind::POST_PREFIX - } else { - kind::POST_SUFFIX - }) else { - return -1; - }; + let metadata = &(unsafe { &*opaque.cast::() }).metadata; let value = if suffix == 0 { - tokens.first() + metadata.bos } else { - tokens.last() + metadata.eos }; let Some(value) = value else { return 1; }; - unsafe { token.write(*value) }; + unsafe { token.write(value) }; 0 })) .unwrap_or(-1) } -/// Return `.tok` config flags, or `u32::MAX` for an invalid handle/image. +/// Return backend flags, or `u32::MAX` for an invalid handle. /// /// # Safety /// -/// `opaque` must be null or a live handle returned by -/// [`tokenizers_hf_create`]. +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`]. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_config_flags(opaque: *const c_void) -> u32 { catch_unwind(AssertUnwindSafe(|| { if opaque.is_null() { return u32::MAX; } - let handle = unsafe { &*opaque.cast::() }; - handle - .file - .reader() - .map(|reader| reader.config.flags) - .unwrap_or(u32::MAX) + (unsafe { &*opaque.cast::() }).metadata.flags })) .unwrap_or(u32::MAX) } +/// Return whether the loaded format carries a decoder, or -1 for an invalid handle. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_has_native_decoder(opaque: *const c_void) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + i32::from( + (unsafe { &*opaque.cast::() }) + .metadata + .native_decoder, + ) + })) + .unwrap_or(-1) +} + /// Destroy a tokenizer handle. /// /// # Safety /// -/// `opaque` must be null or a live handle returned by -/// [`tokenizers_hf_create`] that has not previously been destroyed. +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`] that has not +/// previously been destroyed. #[unsafe(no_mangle)] pub unsafe extern "C" fn tokenizers_hf_destroy(opaque: *mut c_void) { if !opaque.is_null() { @@ -292,9 +622,102 @@ pub unsafe extern "C" fn tokenizers_hf_destroy(opaque: *mut c_void) { mod tests { use super::*; use std::ffi::CString; - use tk_serialization::{Config, Writer, model, pretok, strings}; - fn test_image() -> Vec { + fn temp_path(extension: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-{}-{}.{}", + std::process::id(), + std::thread::current().name().unwrap_or("test"), + extension + )) + } + + fn with_handle(path: &Path, test: impl FnOnce(*mut c_void)) { + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + let handle = unsafe { tokenizers_hf_create(c_path.as_ptr()) }; + assert!(!handle.is_null()); + test(handle); + unsafe { tokenizers_hf_destroy(handle) }; + std::fs::remove_file(path).unwrap(); + } + + fn encoded(handle: *const c_void, text: &[u8]) -> Vec { + let required = unsafe { + tokenizers_hf_encode(handle, text.as_ptr(), text.len(), std::ptr::null_mut(), 0) + }; + assert!(required >= 0); + let mut output = vec![0; required as usize]; + let written = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + output.as_mut_ptr(), + output.len(), + ) + }; + assert_eq!(written, required); + output + } + + #[cfg(feature = "json")] + const TEST_JSON: &str = r#"{ + "version": "2.0", + "role_to_token": {"bos_token": "", "eos_token": ""}, + "added_tokens": [ + {"id": 4, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, + {"id": 5, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true} + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": {"type": "BPE", "byte_level": false, "vocab": {"a": 0, "b": 1, "ab": 2, "abab": 3}, "merges": [["a", "b"], ["ab", "ab"]]}, + "padding": null, + "truncation": null + }"#; + + #[cfg(feature = "json")] + #[test] + fn loads_encodes_and_decodes_json() { + let path = temp_path("json"); + std::fs::write(&path, TEST_JSON).unwrap(); + with_handle(&path, |handle| { + assert_eq!(encoded(handle, b"abab"), [3]); + assert_eq!(unsafe { tokenizers_hf_token_count(handle) }, 6); + assert_eq!(unsafe { tokenizers_hf_config_flags(handle) }, 0); + assert_eq!(unsafe { tokenizers_hf_has_native_decoder(handle) }, 1); + + let mut token = 0; + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 0, &mut token) }, + 0 + ); + assert_eq!(token, 4); + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 1, &mut token) }, + 0 + ); + assert_eq!(token, 5); + + let required = + unsafe { tokenizers_hf_decode(handle, 0, 0, 3, 0, std::ptr::null_mut(), 0) }; + assert_eq!(required, 4); + let mut output = vec![0; required as usize]; + assert_eq!( + unsafe { + tokenizers_hf_decode(handle, 0, 0, 3, 0, output.as_mut_ptr(), output.len()) + }, + required + ); + assert_eq!(output, b"abab"); + }); + } + + #[cfg(feature = "tok")] + fn test_tok_image() -> Vec { + use tk_serialization::{Config, Writer, model, pretok, strings}; + let config = Config { model: model::WORDLEVEL, model_param: 0, @@ -341,94 +764,26 @@ mod tests { writer.finish() } - fn with_handle(test: impl FnOnce(*mut c_void)) { - let path = std::env::temp_dir().join(format!( - "pytorch-tokenizers-hf-{}-{}.tok", - std::process::id(), - std::thread::current().name().unwrap_or("test") - )); - std::fs::write(&path, test_image()).unwrap(); - let c_path = CString::new(path.to_str().unwrap()).unwrap(); - let handle = unsafe { tokenizers_hf_create(c_path.as_ptr()) }; - assert!(!handle.is_null()); - test(handle); - unsafe { tokenizers_hf_destroy(handle) }; - std::fs::remove_file(path).unwrap(); - } - + #[cfg(feature = "tok")] #[test] fn loads_and_encodes_tok() { - with_handle(|handle| { - let text = b"hello"; - let required = unsafe { - tokenizers_hf_encode(handle, text.as_ptr(), text.len(), std::ptr::null_mut(), 0) - }; - assert_eq!(required, 1); - let mut output = [0u32; 1]; - let written = unsafe { - tokenizers_hf_encode( - handle, - text.as_ptr(), - text.len(), - output.as_mut_ptr(), - output.len(), - ) - }; - assert_eq!(written, 1); - assert_eq!(output, [1]); - }); - } - - #[test] - fn exposes_tok_metadata() { - with_handle(|handle| { + let path = temp_path("tok"); + std::fs::write(&path, test_tok_image()).unwrap(); + with_handle(&path, |handle| { + assert_eq!(encoded(handle, b"hello"), [1]); assert_eq!(unsafe { tokenizers_hf_token_count(handle) }, 3); - - let mut id = 0; - let mut text = std::ptr::null(); - let mut len = 0; - let mut added = 0; - let mut special = 0; - assert_eq!( - unsafe { - tokenizers_hf_token_at( - handle, - 2, - &mut id, - &mut text, - &mut len, - &mut added, - &mut special, - ) - }, - 0 - ); - assert_eq!(id, 2); - assert_eq!(unsafe { std::slice::from_raw_parts(text, len) }, b""); - assert_eq!((added, special), (1, 1)); - - let mut token = 0; + assert_eq!(unsafe { tokenizers_hf_has_native_decoder(handle) }, 0); assert_eq!( - unsafe { tokenizers_hf_post_token(handle, 0, &mut token) }, - 0 + unsafe { tokenizers_hf_decode(handle, 0, 0, 1, 0, std::ptr::null_mut(), 0) }, + -1 ); - assert_eq!(token, 2); - assert_eq!( - unsafe { tokenizers_hf_post_token(handle, 1, &mut token) }, - 0 - ); - assert_eq!(token, 2); - assert_eq!(unsafe { tokenizers_hf_config_flags(handle) }, 0); }); } #[test] - fn rejects_non_tok_input() { - let path = std::env::temp_dir().join(format!( - "pytorch-tokenizers-hf-invalid-{}.tok", - std::process::id() - )); - std::fs::write(&path, b"not a tok").unwrap(); + fn rejects_unknown_input() { + let path = temp_path("bin"); + std::fs::write(&path, b"not a tokenizer").unwrap(); let c_path = CString::new(path.to_str().unwrap()).unwrap(); assert!(unsafe { tokenizers_hf_create(c_path.as_ptr()) }.is_null()); std::fs::remove_file(path).unwrap(); diff --git a/src/rust_hf_tokenizer.cpp b/src/rust_hf_tokenizer.cpp index 92a3fd9b..ceca7e29 100644 --- a/src/rust_hf_tokenizer.cpp +++ b/src/rust_hf_tokenizer.cpp @@ -25,6 +25,14 @@ intptr_t tokenizers_hf_encode( size_t text_len, uint32_t* output, size_t output_capacity); +intptr_t tokenizers_hf_decode( + const void* handle, + uint32_t previous, + uint8_t has_previous, + uint32_t token, + uint8_t skip_special, + uint8_t* output, + size_t output_capacity); intptr_t tokenizers_hf_token_count(const void* handle); int32_t tokenizers_hf_token_at( const void* handle, @@ -37,6 +45,7 @@ int32_t tokenizers_hf_token_at( int32_t tokenizers_hf_post_token(const void* handle, uint8_t suffix, uint32_t* token); uint32_t tokenizers_hf_config_flags(const void* handle); +int32_t tokenizers_hf_has_native_decoder(const void* handle); void tokenizers_hf_destroy(void* handle); } @@ -140,22 +149,18 @@ Error RustHFTokenizer::load(const std::string& path) { has_bos_token_ = false; has_eos_token_ = false; byte_level_ = false; + native_decoder_ = false; vocab_size_ = 0; bos_tok_ = 0; eos_tok_ = 0; - std::string tokenizer_tok = path; std::error_code fs_error; - if (fs::is_directory(path, fs_error)) { - const fs::path root(path); - tokenizer_tok = (root / "tokenizer.tok").string(); - } - if (fs_error || !fs::exists(tokenizer_tok, fs_error) || fs_error) { + if (!fs::exists(path, fs_error) || fs_error) { return Error::LoadFailure; } std::unique_ptr handle( - tokenizers_hf_create(tokenizer_tok.c_str())); + tokenizers_hf_create(path.c_str())); if (!handle) { return Error::LoadFailure; } @@ -165,10 +170,13 @@ Error RustHFTokenizer::load(const std::string& path) { return Error::ParseFailure; } byte_level_ = (flags & kByteLevelFlag) != 0; - if (!byte_level_) { - // The v1 .tok format does not serialize decoder configuration. Loading a - // non-byte-level tokenizer would therefore produce raw vocabulary pieces - // instead of decoded text. + const auto native_decoder = tokenizers_hf_has_native_decoder(handle.get()); + if (native_decoder < 0) { + return Error::ParseFailure; + } + native_decoder_ = native_decoder != 0; + if (!byte_level_ && !native_decoder_) { + // The experimental .tok format does not serialize decoder configuration. return Error::LoadFailure; } @@ -364,12 +372,44 @@ Result> RustHFTokenizer::encode( } Result RustHFTokenizer::decode( - uint64_t /*prev_token*/, + uint64_t prev_token, uint64_t token, bool skip_special_tokens) const { if (!initialized_) { return Error::Uninitialized; } + if (!byte_level_) { + if (!native_decoder_ || token > std::numeric_limits::max() || + prev_token > std::numeric_limits::max()) { + return Error::DecodeFailure; + } + auto count = tokenizers_hf_decode( + handle_.get(), + static_cast(prev_token), + prev_token != 0, + static_cast(token), + skip_special_tokens, + nullptr, + 0); + if (count < 0) { + return Error::DecodeFailure; + } + std::string decoded(static_cast(count), '\0'); + count = tokenizers_hf_decode( + handle_.get(), + static_cast(prev_token), + prev_token != 0, + static_cast(token), + skip_special_tokens, + reinterpret_cast(decoded.data()), + decoded.size()); + if (count < 0 || static_cast(count) > decoded.size()) { + return Error::DecodeFailure; + } + decoded.resize(static_cast(count)); + return decoded; + } + std::string_view piece; if (auto regular = token_map_->tryGetString(token)) { piece = *regular; @@ -382,9 +422,6 @@ Result RustHFTokenizer::decode( return Error::DecodeFailure; } - if (!byte_level_) { - return std::string(piece); - } return decode_byte_level(piece); } diff --git a/test/test_rust_hf_tokenizer.cpp b/test/test_rust_hf_tokenizer.cpp index 5a02b615..f27f84b3 100644 --- a/test/test_rust_hf_tokenizer.cpp +++ b/test/test_rust_hf_tokenizer.cpp @@ -22,6 +22,7 @@ namespace { constexpr uint32_t kByteLevelFlag = 1U << 2; uint32_t g_config_flags = kByteLevelFlag; +bool g_native_decoder = false; bool g_use_known_eos = true; struct TokenRecord { @@ -81,6 +82,25 @@ intptr_t tokenizers_hf_encode( return encoded.size(); } +intptr_t tokenizers_hf_decode( + const void*, + uint32_t, + uint8_t, + uint32_t token, + uint8_t skip_special, + uint8_t* output, + size_t output_capacity) { + const std::string decoded = + skip_special != 0 && token == 2 ? "" : "decoded world"; + if (output_capacity < decoded.size()) { + return decoded.size(); + } + if (!decoded.empty()) { + std::copy(decoded.begin(), decoded.end(), output); + } + return decoded.size(); +} + intptr_t tokenizers_hf_token_count(const void*) { return kRecords.size(); } @@ -120,6 +140,10 @@ uint32_t tokenizers_hf_config_flags(const void*) { return g_config_flags; } +int32_t tokenizers_hf_has_native_decoder(const void*) { + return g_native_decoder; +} + void tokenizers_hf_destroy(void* handle) { delete static_cast(handle); } @@ -131,6 +155,7 @@ namespace { TEST(RustHFTokenizerTest, PreservesIndependentBosAndEosCounts) { g_config_flags = kByteLevelFlag; + g_native_decoder = false; g_use_known_eos = true; TemporaryTokFile file; RustHFTokenizer tokenizer; @@ -149,6 +174,7 @@ TEST(RustHFTokenizerTest, PreservesIndependentBosAndEosCounts) { TEST(RustHFTokenizerTest, DecodesByteLevelPieces) { g_config_flags = kByteLevelFlag; + g_native_decoder = false; g_use_known_eos = true; TemporaryTokFile file; RustHFTokenizer tokenizer; @@ -163,11 +189,31 @@ TEST(RustHFTokenizerTest, DecodesByteLevelPieces) { EXPECT_TRUE(skipped->empty()); } +TEST(RustHFTokenizerTest, UsesNativeDecoderForJson) { + g_config_flags = 0; + g_native_decoder = true; + g_use_known_eos = true; + TemporaryTokFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + + auto decoded = tokenizer.decode(3, 4); + ASSERT_TRUE(decoded.ok()); + EXPECT_EQ(*decoded, "decoded world"); + + auto skipped = tokenizer.decode(4, 2, true); + ASSERT_TRUE(skipped.ok()); + EXPECT_TRUE(skipped->empty()); + g_config_flags = kByteLevelFlag; + g_native_decoder = false; +} + TEST(RustHFTokenizerTest, RejectsTokWithoutSupportedDecoder) { g_use_known_eos = true; TemporaryTokFile file; RustHFTokenizer tokenizer; g_config_flags = 0; + g_native_decoder = false; EXPECT_EQ(tokenizer.load(file.string()), Error::LoadFailure); EXPECT_FALSE(tokenizer.is_loaded()); g_config_flags = kByteLevelFlag; From 48970bb564014e075a0c05cda787cebcb3bbc1fd Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 13:45:40 -0700 Subject: [PATCH 4/4] Refresh tokenizer size measurements Generated with assistance from Codex. --- rust_tokenizer/binary-size.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rust_tokenizer/binary-size.md b/rust_tokenizer/binary-size.md index da9dea8d..a1882450 100644 --- a/rust_tokenizer/binary-size.md +++ b/rust_tokenizer/binary-size.md @@ -1,16 +1,16 @@ # Hugging Face Rust tokenizer backend binary size Measured on Apple arm64 with Rust 1.98.1, `CMAKE_BUILD_TYPE=Release`, -`TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. The smoke binary -loads a GPT-2 `.tok`, encodes `Hello world`, performs vocabulary lookups, and -decodes both output tokens. +`TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. Each smoke binary +loads the corresponding GPT-2 artifact, encodes `Hello world`, performs +vocabulary lookups, and decodes both output tokens. | Configuration | Stripped | Gzipped | |---|---:|---:| | Default (`TOKENIZERS_BUILD_HF_RUST_TOKENIZER=OFF`) | 0 B added | 0 B added | -| `.tok` only | 817,888 B | 377,292 B | -| JSON only | 1,921,824 B | 902,655 B | -| JSON and `.tok` | 2,105,024 B | 1,007,448 B | +| `.tok` only | 694,768 B | 353,651 B | +| JSON only | 1,921,872 B | 901,688 B | +| JSON and `.tok` | 2,105,056 B | 1,006,186 B | The `.tok` row uses `TOKENIZERS_HF_RUST_FORMATS=tok`; it does not link the JSON reader or canonicalizer. The JSON row uses the published v1.0.0-rc.2 crates.