diff --git a/CMakeLists.txt b/CMakeLists.txt index 62ca12cf..66a5dd56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,12 @@ project(Tokenizers) option(TOKENIZERS_BUILD_TEST "Build tests" OFF) option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF) option(TOKENIZERS_BUILD_PYTHON "Build Python bindings" OFF) +option(TOKENIZERS_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face Rust tokenizer backend" OFF +) +option(TOKENIZERS_OPTIMIZE_SIZE "Optimize optional tokenizer backends for size" + OFF +) option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF ) @@ -201,6 +207,10 @@ endif() # Installation rules include(GNUInstallDirs) +if(TOKENIZERS_BUILD_HF_RUST_TOKENIZER) + add_subdirectory(rust_tokenizer) +endif() + if(NOT TOKENIZERS_BUILD_PYTHON) # Install the library and its dependencies install( diff --git a/cmake/tokenizers-config.cmake.in b/cmake/tokenizers-config.cmake.in index 1e9f87b2..dc583d66 100644 --- a/cmake/tokenizers-config.cmake.in +++ b/cmake/tokenizers-config.cmake.in @@ -25,6 +25,25 @@ endif() find_dependency(re2 REQUIRED) find_dependency(absl REQUIRED) +# The optional Hugging Face backend is a C++ archive with a Rust static-library +# dependency. Define the latter before importing the exported targets when the +# archive is present. Merely finding the package does not link either target. +if(WIN32) + set(_TOKENIZERS_HF_FFI_NAME "tokenizers_hf_ffi.lib") +else() + set(_TOKENIZERS_HF_FFI_NAME "libtokenizers_hf_ffi.a") +endif() +set(_TOKENIZERS_HF_FFI "${TOKENIZERS_LIBDIR}/${_TOKENIZERS_HF_FFI_NAME}") +if(EXISTS "${_TOKENIZERS_HF_FFI}" AND NOT TARGET tokenizers_hf_ffi) + if(UNIX AND NOT APPLE) + find_dependency(Threads) + endif() + add_library(tokenizers_hf_ffi STATIC IMPORTED) + set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION "${_TOKENIZERS_HF_FFI}" + ) +endif() + # Include the exported targets file include("${CMAKE_CURRENT_LIST_DIR}/tokenizers-targets.cmake") diff --git a/include/pytorch/tokenizers/rust_hf_tokenizer.h b/include/pytorch/tokenizers/rust_hf_tokenizer.h new file mode 100644 index 00000000..7699bb8b --- /dev/null +++ b/include/pytorch/tokenizers/rust_hf_tokenizer.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace tokenizers { + +class RustHFTokenizer final : public Tokenizer { + public: + RustHFTokenizer(); + ~RustHFTokenizer() override; + + Error load(const std::string& tokenizer_path) override; + Result id_to_piece(uint64_t token) const override; + Result piece_to_id(const std::string& text) const override; + Result> encode( + const std::string& input, + int8_t bos = 0, + int8_t eos = 0) const override; + Result decode( + uint64_t prev_token, + uint64_t token, + bool skip_special_tokens = false) const override; + + private: + using TokenMap = detail::StringIntegerMap<>; + + struct RustHandleDeleter { + void operator()(void* handle) const; + }; + + Error load_metadata(const void* handle); + + std::unique_ptr handle_; + std::optional token_map_; + std::optional added_token_map_; + std::unordered_set special_token_ids_; + bool has_bos_token_ = false; + bool has_eos_token_ = false; + bool byte_level_ = false; +}; + +} // namespace tokenizers diff --git a/rust_tokenizer/.gitignore b/rust_tokenizer/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/rust_tokenizer/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/rust_tokenizer/CMakeLists.txt b/rust_tokenizer/CMakeLists.txt new file mode 100644 index 00000000..a9bc4831 --- /dev/null +++ b/rust_tokenizer/CMakeLists.txt @@ -0,0 +1,71 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +find_program(CARGO_EXECUTABLE cargo REQUIRED) + +set(_rust_profile release) +if(TOKENIZERS_OPTIMIZE_SIZE) + set(_rust_profile minsize) +endif() + +set(_cargo_target_dir ${CMAKE_CURRENT_BINARY_DIR}/cargo-target) +string(CONCAT _rust_library_name ${CMAKE_STATIC_LIBRARY_PREFIX} + tokenizers_hf_ffi ${CMAKE_STATIC_LIBRARY_SUFFIX} +) +set(_rust_library ${_cargo_target_dir}/${_rust_profile}/${_rust_library_name}) +file(GLOB_RECURSE _rust_sources CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/src/*.rs +) + +add_custom_command( + OUTPUT ${_rust_library} + COMMAND + ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${_cargo_target_dir} + CARGO_ENCODED_RUSTFLAGS=-Crelocation-model=pic ${CARGO_EXECUTABLE} build + --locked --manifest-path ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml --profile + ${_rust_profile} + DEPENDS ${_rust_sources} ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml + ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM +) + +add_custom_target(tokenizers_hf_ffi_build DEPENDS ${_rust_library}) +add_library(tokenizers_hf_ffi STATIC IMPORTED GLOBAL) +set_target_properties( + tokenizers_hf_ffi PROPERTIES IMPORTED_LOCATION ${_rust_library} +) +add_dependencies(tokenizers_hf_ffi tokenizers_hf_ffi_build) + +add_library(tokenizers_hf_rust_tokenizer STATIC ../src/rust_hf_tokenizer.cpp) +add_library(tokenizers::hf_rust_tokenizer ALIAS tokenizers_hf_rust_tokenizer) +set_target_properties( + tokenizers_hf_rust_tokenizer PROPERTIES EXPORT_NAME hf_rust_tokenizer +) +target_include_directories( + tokenizers_hf_rust_tokenizer + PUBLIC $ + $ +) +target_compile_features(tokenizers_hf_rust_tokenizer PUBLIC cxx_std_17) +target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE tokenizers_hf_ffi) + +if(APPLE) + target_link_libraries(tokenizers_hf_rust_tokenizer PRIVATE iconv) +elseif(UNIX) + find_package(Threads REQUIRED) + target_link_libraries( + tokenizers_hf_rust_tokenizer PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m rt + util + ) +endif() + +install( + TARGETS tokenizers_hf_rust_tokenizer + EXPORT tokenizers-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +install(FILES ${_rust_library} DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/rust_tokenizer/Cargo.lock b/rust_tokenizer/Cargo.lock new file mode 100644 index 00000000..239d8c4a --- /dev/null +++ b/rust_tokenizer/Cargo.lock @@ -0,0 +1,1293 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "arbitrary-chunks" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitcannon" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfc3924e96bdcf270ede3ff8e0713b68edaf2daa3ed1661467ed58cc5630fba" +dependencies = [ + "ahash", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hifijson" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "242402749acf71e6f32f5857598b7002c4058a4e3c3b22b4c7d51cab9aea754e" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "mem_dbg" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48a1086c746f4ee6ca5cb0acf856a14709bc4d2d20e03db150a12ddf2269e6d" +dependencies = [ + "bitflags", + "hashbrown", + "mem_dbg-derive", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb910efe8da52f13da727170e352e50a1764579a6fb1065d00d9556da19c79ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "partition" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefetch-index" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d217d8cc2d882b70ba0602224621e35039c7f4c09f158ba8a560be9cd48ff7" +dependencies = [ + "bitvec", + "colored", + "fastrand", + "fxhash", + "itertools 0.15.0", + "log", + "mem_dbg", + "prefetch-index", + "rand", + "rand_chacha", + "rayon", + "rdst", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +dependencies = [ + "serde_json", + "tk-convert", + "tk-encode", + "tk-serialize", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdst" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +dependencies = [ + "arbitrary-chunks", + "block-pseudorand", + "criterion", + "partition", + "rayon", + "tikv-jemallocator", + "voracious_radix_sort", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "safe_arch" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tk-convert" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae9aa847b762e874fe27be83b120f63fc76fd7f555d254540d16e044660dfeb" +dependencies = [ + "bitcannon", + "serde_json", + "thiserror", +] + +[[package]] +name = "tk-encode" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b28a17e23359d88085a838d0c893875f7d220262368eeafdfd662577b1ba9ee" +dependencies = [ + "ahash", + "bitcannon", + "dary_heap", + "fancy-regex", + "itertools 0.15.0", + "log", + "memchr", + "paste", + "ptr_hash", + "rand", + "spm_precompiled", + "thiserror", + "unicode-normalization", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", + "wide", + "yada", +] + +[[package]] +name = "tk-serialize" +version = "0.1.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39d1d911331f6c6e6a626bb4a67b3ddc86bceab6a5ee50918b1009bbda668fe" +dependencies = [ + "base64 0.22.1", + "bitcannon", + "hifijson", + "tk-encode", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wide" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d920ac99c3c8edce110cb8d07dbb324d6d026011dce85b1e9355b70f0adacc4f" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yada" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" + +[[package]] +name = "zerocopy" +version = "0.8.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e8fafad82b542ff3717217ecdc736231b59e387768c9630123b4ce4d2db44" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595f56e044df4f46a0c9a626f65c3d99eb8488f7e8a8baa12dd76326d9710bf2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust_tokenizer/Cargo.toml b/rust_tokenizer/Cargo.toml new file mode 100644 index 00000000..a58619ec --- /dev/null +++ b/rust_tokenizer/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "pytorch-tokenizers-hf-ffi" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +name = "tokenizers_hf_ffi" +crate-type = ["staticlib"] + +[dependencies] +serde_json = "1.0" +tk-convert = { version = "=0.1.0-rc.2", default-features = false } +tk-encode = { version = "=0.1.0-rc.2", default-features = false, features = [ + "bpe", + "fancy-regex", + "normalizers", + "unicode-scripts", + "unigram", + "wordlevel", + "wordpiece", +] } +tk-serialize = { version = "=0.1.0-rc.2", default-features = false, features = [ + "bpe", + "deserialize", + "fancy-regex", + "normalizers", + "unicode-scripts", + "unigram", + "wordlevel", + "wordpiece", +] } +[profile.release] +codegen-units = 1 +lto = "fat" +opt-level = 3 +strip = true + +[profile.minsize] +inherits = "release" +opt-level = "z" +panic = "abort" diff --git a/rust_tokenizer/README.md b/rust_tokenizer/README.md new file mode 100644 index 00000000..7ad9c5a3 --- /dev/null +++ b/rust_tokenizer/README.md @@ -0,0 +1,32 @@ +# Hugging Face Rust tokenizer backend + +This backend is opt-in and disabled by default. It loads standard Hugging Face +`tokenizer.json` files through the published v1 release-candidate crates. Enable +it with `-DTOKENIZERS_BUILD_HF_RUST_TOKENIZER=ON`. When disabled, CMake does not +invoke Cargo or compile the C++ bridge, so the default tokenizer library is +unchanged. ExecuTorch users also set +`-DEXECUTORCH_BUILD_EXTENSION_LLM=ON`. + +Use `-DTOKENIZERS_OPTIMIZE_SIZE=ON` for the Rust `minsize` profile; ExecuTorch +forwards `EXECUTORCH_OPTIMIZE_SIZE` to this option. + +The backend uses Hugging Face tokenizers v1.0.0-rc.2's `tk-encode`, +`tk-serialize`, and `tk-convert` crates. It accepts normal Hub +`tokenizer.json` files, canonicalizes legacy v1 JSON in memory, and uses the +upstream decoder chain. If a directory is passed, it reads `tokenizer.json` and +uses sibling `tokenizer_config.json` or `special_tokens_map.json` for BOS/EOS +metadata when available. + +The adapter requires identifiable BOS and EOS IDs because the ExecuTorch +tokenizer interface cannot represent either value as absent. If the Rust +backend cannot load a JSON file, the LLM runner retains its existing C++ +`HFTokenizer` fallback. + +This integration currently supports host CMake builds. Android, Apple +framework, WASM, and Buck packaging still need explicit Rust target/toolchain +integration. The build requires Cargo and fetches its pinned crates.io +dependencies on the first run. + +Hugging Face has announced inference-only C and C++ bindings for ExecuTorch and +llama.cpp on the v1 roadmap. The local C ABI is deliberately small so the +implementation can move to that upstream binding when it stabilizes. diff --git a/rust_tokenizer/binary-size.md b/rust_tokenizer/binary-size.md new file mode 100644 index 00000000..001e8716 --- /dev/null +++ b/rust_tokenizer/binary-size.md @@ -0,0 +1,15 @@ +# Hugging Face Rust tokenizer backend binary size + +Measured on Apple arm64 with Rust 1.98.1, `CMAKE_BUILD_TYPE=Release`, +`TOKENIZERS_OPTIMIZE_SIZE=ON`, dead stripping, and `gzip -9`. The smoke binary +loads a GPT-2 `tokenizer.json`, encodes `Hello world`, performs vocabulary +lookups, and decodes both output tokens. + +| Configuration | Stripped | Gzipped | +|---|---:|---:| +| Default (`TOKENIZERS_BUILD_HF_RUST_TOKENIZER=OFF`) | 0 B added | 0 B added | +| JSON backend enabled | 1,921,872 B | 901,688 B | + +This is a complete smoke executable that loads and round-trips GPT-2, not an +archive size. The OFF configuration exposes no Rust CMake target and produces +no Cargo build directory. diff --git a/rust_tokenizer/src/lib.rs b/rust_tokenizer/src/lib.rs new file mode 100644 index 00000000..c0567d9a --- /dev/null +++ b/rust_tokenizer/src/lib.rs @@ -0,0 +1,591 @@ +use std::ffi::{CStr, c_char, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; + +use serde_json::Value; +use tk_encode::pipeline::{EncodeOptions, Override, PipelineModel, PipelineTokenizer}; + +const BYTE_LEVEL_FLAG: u32 = 1 << 2; + +struct TokenRecord { + id: u32, + text: String, + added: bool, + special: bool, +} + +struct Metadata { + records: Vec, + bos: Option, + eos: Option, + flags: u32, +} + +struct Handle { + tokenizer: PipelineTokenizer, + metadata: Metadata, +} + +impl Handle { + fn load(path: &Path) -> Result { + if path.extension().and_then(|extension| extension.to_str()) != Some("json") { + return Err(()); + } + let canonical = tk_convert::canonicalize_file(path).map_err(|_| ())?; + let tokenizer = tk_serialize::from_json(&canonical).map_err(|_| ())?; + let mut metadata = json_metadata(&canonical)?; + + let post_processor = tokenizer.get_post_processor(); + let post_bos = post_processor + .single + .prefix + .first() + .map(|(token, _)| token.id()); + let post_eos = post_processor + .single + .suffix + .last() + .map(|(token, _)| token.id()); + let role_bos = tokenizer + .get_token_for_role("bos_token") + .and_then(|token| token_id(&metadata.records, token)); + let role_eos = tokenizer + .get_token_for_role("eos_token") + .and_then(|token| token_id(&metadata.records, token)); + let sidecar = sidecar_special_tokens(path); + metadata.bos = role_bos + .or_else(|| { + sidecar + .0 + .and_then(|token| token_id(&metadata.records, &token)) + }) + .or(post_bos); + metadata.eos = role_eos + .or_else(|| { + sidecar + .1 + .and_then(|token| token_id(&metadata.records, &token)) + }) + .or(post_eos); + if matches!( + tokenizer.get_model(), + PipelineModel::BPE(model) if model.is_byte_level() + ) { + metadata.flags |= BYTE_LEVEL_FLAG; + } + Ok(Self { + tokenizer, + metadata, + }) + } + + fn encode(&self, text: &str) -> Result, ()> { + let options = EncodeOptions { + padding: Override::Off, + truncation: Override::Off, + ..EncodeOptions::no_specials() + }; + let mut encodings = self + .tokenizer + .encode(text, &options) + .wait() + .map_err(|_| ())?; + if encodings.len() != 1 { + return Err(()); + } + Ok(encodings + .pop() + .expect("one encoding") + .ids() + .iter() + .map(|token| token.id()) + .collect()) + } + + fn decode(&self, previous: Option, token: u32, skip_special: bool) -> Result { + let current = self + .tokenizer + .decode(&[token], skip_special) + .map_err(|_| ())?; + let Some(previous) = previous else { + return Ok(current); + }; + let prefix = self + .tokenizer + .decode(&[previous], skip_special) + .map_err(|_| ())?; + let pair = self + .tokenizer + .decode(&[previous, token], skip_special) + .map_err(|_| ())?; + Ok(pair.strip_prefix(&prefix).unwrap_or(¤t).to_string()) + } +} + +fn token_id(records: &[TokenRecord], text: &str) -> Option { + records + .iter() + .find(|record| record.text == text) + .map(|record| record.id) +} + +fn json_metadata(text: &str) -> Result { + let document: Value = serde_json::from_str(text).map_err(|_| ())?; + let model = document.get("model").and_then(Value::as_object).ok_or(())?; + let vocab = model.get("vocab").ok_or(())?; + let mut records = Vec::new(); + if let Some(entries) = vocab.as_object() { + for (token, id) in entries { + records.push(TokenRecord { + id: value_u32(id)?, + text: token.clone(), + added: false, + special: false, + }); + } + } else if let Some(entries) = vocab.as_array() { + for (id, entry) in entries.iter().enumerate() { + let token = entry + .as_array() + .and_then(|fields| fields.first()) + .and_then(Value::as_str) + .ok_or(())?; + records.push(TokenRecord { + id: u32::try_from(id).map_err(|_| ())?, + text: token.to_owned(), + added: false, + special: false, + }); + } + } else { + return Err(()); + } + + if let Some(added_tokens) = document.get("added_tokens").and_then(Value::as_array) { + for token in added_tokens { + records.push(TokenRecord { + id: value_u32(token.get("id").ok_or(())?)?, + text: token + .get("content") + .and_then(Value::as_str) + .ok_or(())? + .to_owned(), + added: true, + special: token + .get("special") + .and_then(Value::as_bool) + .unwrap_or(false), + }); + } + } + + let roles = document.get("role_to_token").and_then(Value::as_object); + let bos = roles + .and_then(|roles| roles.get("bos_token")) + .and_then(special_token_text) + .and_then(|token| token_id(&records, token)); + let eos = roles + .and_then(|roles| roles.get("eos_token")) + .and_then(special_token_text) + .and_then(|token| token_id(&records, token)); + Ok(Metadata { + records, + bos, + eos, + flags: 0, + }) +} + +fn value_u32(value: &Value) -> Result { + value + .as_u64() + .and_then(|id| u32::try_from(id).ok()) + .ok_or(()) +} + +fn special_token_text(value: &Value) -> Option<&str> { + value + .as_str() + .or_else(|| value.get("content").and_then(Value::as_str)) +} + +fn sidecar_special_tokens(path: &Path) -> (Option, Option) { + let mut bos = None; + let mut eos = None; + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + for filename in ["tokenizer_config.json", "special_tokens_map.json"] { + let Ok(text) = std::fs::read_to_string(parent.join(filename)) else { + continue; + }; + let Ok(document) = serde_json::from_str::(&text) else { + continue; + }; + if bos.is_none() { + bos = document + .get("bos_token") + .and_then(special_token_text) + .map(str::to_owned); + } + if eos.is_none() { + eos = document + .get("eos_token") + .and_then(special_token_text) + .map(str::to_owned); + } + } + (bos, eos) +} + +fn candidate_paths(path: &Path) -> Vec { + if !path.is_dir() { + return vec![path.to_owned()]; + } + vec![path.join("tokenizer.json")] +} + +/// Create a tokenizer from a `tokenizer.json` or directory containing one. +/// +/// # Safety +/// +/// `path` must be null or point to a valid NUL-terminated C string for the duration of this call. +/// A non-null return value must eventually be passed exactly once to [`tokenizers_hf_destroy`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_create(path: *const c_char) -> *mut c_void { + catch_unwind(AssertUnwindSafe(|| { + if path.is_null() { + return std::ptr::null_mut(); + } + let Ok(path) = unsafe { CStr::from_ptr(path) }.to_str() else { + return std::ptr::null_mut(); + }; + for candidate in candidate_paths(Path::new(path)) { + if let Ok(handle) = Handle::load(&candidate) { + return Box::into_raw(Box::new(handle)).cast(); + } + } + std::ptr::null_mut() + })) + .unwrap_or(std::ptr::null_mut()) +} + +/// Encode one UTF-8 string into caller-owned token storage. +/// +/// # Safety +/// +/// `opaque` must be a live handle returned by [`tokenizers_hf_create`]. `text` must address +/// `text_len` readable bytes unless the length is zero. `output` must address `output_capacity` +/// writable `u32` values unless the capacity is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_encode( + opaque: *const c_void, + text: *const u8, + text_len: usize, + output: *mut u32, + output_capacity: usize, +) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || (text.is_null() && text_len != 0) { + return -1; + } + let bytes = if text_len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(text, text_len) } + }; + let Ok(text) = std::str::from_utf8(bytes) else { + return -1; + }; + let Ok(tokens) = (unsafe { &*opaque.cast::() }).encode(text) else { + return -1; + }; + let Ok(token_count) = isize::try_from(tokens.len()) else { + return -1; + }; + if tokens.len() > output_capacity { + return token_count; + } + if !tokens.is_empty() && output.is_null() { + return -1; + } + if !tokens.is_empty() { + unsafe { std::ptr::copy_nonoverlapping(tokens.as_ptr(), output, tokens.len()) }; + } + token_count + })) + .unwrap_or(-1) +} + +/// Decode one token with optional previous-token context into caller-owned byte storage. +/// +/// `has_previous == 0` ignores `previous`. The return value is the required byte count, or -1 +/// when decoding fails. +/// +/// # Safety +/// +/// `opaque` must be a live handle. `output` must address `output_capacity` writable bytes unless +/// the capacity is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_decode( + opaque: *const c_void, + previous: u32, + has_previous: u8, + token: u32, + skip_special: u8, + output: *mut u8, + output_capacity: usize, +) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + let previous = (has_previous != 0).then_some(previous); + let Ok(decoded) = + (unsafe { &*opaque.cast::() }).decode(previous, token, skip_special != 0) + else { + return -1; + }; + let Ok(byte_count) = isize::try_from(decoded.len()) else { + return -1; + }; + if decoded.len() > output_capacity { + return byte_count; + } + if !decoded.is_empty() && output.is_null() { + return -1; + } + if !decoded.is_empty() { + unsafe { + std::ptr::copy_nonoverlapping(decoded.as_ptr(), output, decoded.len()); + } + } + byte_count + })) + .unwrap_or(-1) +} + +/// Return the number of vocabulary and added-token records. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_count(opaque: *const c_void) -> isize { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return -1; + } + isize::try_from( + (unsafe { &*opaque.cast::() }) + .metadata + .records + .len(), + ) + .unwrap_or(-1) + })) + .unwrap_or(-1) +} + +/// Read one vocabulary record. Returned string bytes borrow `opaque` and remain valid until the +/// handle is destroyed. +/// +/// # Safety +/// +/// `opaque` must be a live handle. Every output argument must point to writable storage. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_token_at( + opaque: *const c_void, + index: usize, + id: *mut u32, + text: *mut *const u8, + text_len: *mut usize, + is_added: *mut u8, + is_special: *mut u8, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() + || id.is_null() + || text.is_null() + || text_len.is_null() + || is_added.is_null() + || is_special.is_null() + { + return -1; + } + let handle = unsafe { &*opaque.cast::() }; + let Some(record) = handle.metadata.records.get(index) else { + return -1; + }; + unsafe { + id.write(record.id); + text.write(record.text.as_ptr()); + text_len.write(record.text.len()); + is_added.write(u8::from(record.added)); + is_special.write(u8::from(record.special)); + } + 0 + })) + .unwrap_or(-1) +} + +/// Return the configured BOS (`suffix == 0`) or EOS (`suffix != 0`) token. +/// +/// # Safety +/// +/// `opaque` must be a live handle and `token` must point to writable storage. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_post_token( + opaque: *const c_void, + suffix: u8, + token: *mut u32, +) -> i32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() || token.is_null() { + return -1; + } + let metadata = &(unsafe { &*opaque.cast::() }).metadata; + let value = if suffix == 0 { + metadata.bos + } else { + metadata.eos + }; + let Some(value) = value else { + return 1; + }; + unsafe { token.write(value) }; + 0 + })) + .unwrap_or(-1) +} + +/// Return backend flags, or `u32::MAX` for an invalid handle. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_config_flags(opaque: *const c_void) -> u32 { + catch_unwind(AssertUnwindSafe(|| { + if opaque.is_null() { + return u32::MAX; + } + (unsafe { &*opaque.cast::() }).metadata.flags + })) + .unwrap_or(u32::MAX) +} + +/// Destroy a tokenizer handle. +/// +/// # Safety +/// +/// `opaque` must be null or a live handle returned by [`tokenizers_hf_create`] that has not +/// previously been destroyed. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tokenizers_hf_destroy(opaque: *mut c_void) { + if !opaque.is_null() { + let _ = catch_unwind(AssertUnwindSafe(|| { + drop(unsafe { Box::from_raw(opaque.cast::()) }); + })); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + fn temp_path(extension: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "pytorch-tokenizers-hf-{}-{}.{}", + std::process::id(), + std::thread::current().name().unwrap_or("test"), + extension + )) + } + + fn with_handle(path: &Path, test: impl FnOnce(*mut c_void)) { + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + let handle = unsafe { tokenizers_hf_create(c_path.as_ptr()) }; + assert!(!handle.is_null()); + test(handle); + unsafe { tokenizers_hf_destroy(handle) }; + std::fs::remove_file(path).unwrap(); + } + + fn encoded(handle: *const c_void, text: &[u8]) -> Vec { + let required = unsafe { + tokenizers_hf_encode(handle, text.as_ptr(), text.len(), std::ptr::null_mut(), 0) + }; + assert!(required >= 0); + let mut output = vec![0; required as usize]; + let written = unsafe { + tokenizers_hf_encode( + handle, + text.as_ptr(), + text.len(), + output.as_mut_ptr(), + output.len(), + ) + }; + assert_eq!(written, required); + output + } + + const TEST_JSON: &str = r#"{ + "version": "2.0", + "role_to_token": {"bos_token": "", "eos_token": ""}, + "added_tokens": [ + {"id": 4, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, + {"id": 5, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true} + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": {"type": "BPE", "byte_level": false, "vocab": {"a": 0, "b": 1, "ab": 2, "abab": 3}, "merges": [["a", "b"], ["ab", "ab"]]}, + "padding": null, + "truncation": null + }"#; + + #[test] + fn loads_encodes_and_decodes_json() { + let path = temp_path("json"); + std::fs::write(&path, TEST_JSON).unwrap(); + with_handle(&path, |handle| { + assert_eq!(encoded(handle, b"abab"), [3]); + assert_eq!(unsafe { tokenizers_hf_token_count(handle) }, 6); + assert_eq!(unsafe { tokenizers_hf_config_flags(handle) }, 0); + let mut token = 0; + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 0, &mut token) }, + 0 + ); + assert_eq!(token, 4); + assert_eq!( + unsafe { tokenizers_hf_post_token(handle, 1, &mut token) }, + 0 + ); + assert_eq!(token, 5); + + let required = + unsafe { tokenizers_hf_decode(handle, 0, 0, 3, 0, std::ptr::null_mut(), 0) }; + assert_eq!(required, 4); + let mut output = vec![0; required as usize]; + assert_eq!( + unsafe { + tokenizers_hf_decode(handle, 0, 0, 3, 0, output.as_mut_ptr(), output.len()) + }, + required + ); + assert_eq!(output, b"abab"); + }); + } + + #[test] + fn rejects_unknown_input() { + let path = temp_path("bin"); + std::fs::write(&path, b"not a tokenizer").unwrap(); + let c_path = CString::new(path.to_str().unwrap()).unwrap(); + assert!(unsafe { tokenizers_hf_create(c_path.as_ptr()) }.is_null()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/src/rust_hf_tokenizer.cpp b/src/rust_hf_tokenizer.cpp new file mode 100644 index 00000000..6e151aac --- /dev/null +++ b/src/rust_hf_tokenizer.cpp @@ -0,0 +1,416 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +extern "C" { +void* tokenizers_hf_create(const char* path); +intptr_t tokenizers_hf_encode( + const void* handle, + const uint8_t* text, + size_t text_len, + uint32_t* output, + size_t output_capacity); +intptr_t tokenizers_hf_decode( + const void* handle, + uint32_t previous, + uint8_t has_previous, + uint32_t token, + uint8_t skip_special, + uint8_t* output, + size_t output_capacity); +intptr_t tokenizers_hf_token_count(const void* handle); +int32_t tokenizers_hf_token_at( + const void* handle, + size_t index, + uint32_t* id, + const uint8_t** text, + size_t* text_len, + uint8_t* is_added, + uint8_t* is_special); +int32_t +tokenizers_hf_post_token(const void* handle, uint8_t suffix, uint32_t* token); +uint32_t tokenizers_hf_config_flags(const void* handle); +void tokenizers_hf_destroy(void* handle); +} + +namespace tokenizers { +namespace { + +// Mirrors the byte-level flag returned by the Rust pipeline metadata. +constexpr uint32_t kByteLevelFlag = 1U << 2; + +bool is_bos_token(std::string_view token) { + return token == "" || token == "[CLS]" || token == "" || + token == "" || token == "<|bos|>" || token == "<|begin_of_text|>" || + token == "<|start_of_text|>" || token == "<|startoftext|>" || + token == "<|endoftext|>"; +} + +bool is_eos_token(std::string_view token) { + return token == "" || token == "[SEP]" || token == "" || + token == "" || token == "<|eos|>" || token == "<|end_of_text|>" || + token == "<|endoftext|>"; +} + +std::optional byte_level_codepoint_to_byte(uint32_t codepoint) { + if ((codepoint >= 33 && codepoint <= 126) || + (codepoint >= 161 && codepoint <= 172) || + (codepoint >= 174 && codepoint <= 255)) { + return static_cast(codepoint); + } + if (codepoint < 256 || codepoint > 323) { + return std::nullopt; + } + const auto index = codepoint - 256; + if (index < 33) { + return static_cast(index); + } + if (index < 67) { + return static_cast(127 + index - 33); + } + return static_cast(173); +} + +std::string decode_byte_level(std::string_view piece) { + std::string decoded; + decoded.reserve(piece.size()); + for (size_t index = 0; index < piece.size();) { + const auto first = static_cast(piece[index]); + uint32_t codepoint = 0; + size_t length = 0; + if ((first & 0x80) == 0) { + codepoint = first; + length = 1; + } else if ((first & 0xE0) == 0xC0) { + codepoint = first & 0x1F; + length = 2; + } else if ((first & 0xF0) == 0xE0) { + codepoint = first & 0x0F; + length = 3; + } else if ((first & 0xF8) == 0xF0) { + codepoint = first & 0x07; + length = 4; + } else { + return std::string(piece); + } + if (length > piece.size() - index) { + return std::string(piece); + } + for (size_t offset = 1; offset < length; ++offset) { + const auto continuation = static_cast(piece[index + offset]); + if ((continuation & 0xC0) != 0x80) { + return std::string(piece); + } + codepoint = (codepoint << 6) | (continuation & 0x3F); + } + const auto byte = byte_level_codepoint_to_byte(codepoint); + if (!byte) { + return std::string(piece); + } + decoded.push_back(static_cast(*byte)); + index += length; + } + return decoded; +} + +} // namespace + +RustHFTokenizer::RustHFTokenizer() : handle_(nullptr) {} + +RustHFTokenizer::~RustHFTokenizer() = default; + +void RustHFTokenizer::RustHandleDeleter::operator()(void* handle) const { + tokenizers_hf_destroy(handle); +} + +Error RustHFTokenizer::load(const std::string& path) { + initialized_ = false; + handle_.reset(); + token_map_.reset(); + added_token_map_.reset(); + special_token_ids_.clear(); + has_bos_token_ = false; + has_eos_token_ = false; + byte_level_ = false; + vocab_size_ = 0; + bos_tok_ = 0; + eos_tok_ = 0; + + std::error_code fs_error; + if (!fs::exists(path, fs_error) || fs_error) { + return Error::LoadFailure; + } + + std::unique_ptr handle( + tokenizers_hf_create(path.c_str())); + if (!handle) { + return Error::LoadFailure; + } + + const auto flags = tokenizers_hf_config_flags(handle.get()); + if (flags == std::numeric_limits::max()) { + return Error::ParseFailure; + } + byte_level_ = (flags & kByteLevelFlag) != 0; + + const auto metadata_error = load_metadata(handle.get()); + if (metadata_error != Error::Ok) { + return metadata_error; + } + + handle_ = std::move(handle); + initialized_ = true; + return Error::Ok; +} + +Error RustHFTokenizer::load_metadata(const void* handle) { + const auto count = tokenizers_hf_token_count(handle); + if (count < 0 || + static_cast(count) > + static_cast(std::numeric_limits::max())) { + return Error::ParseFailure; + } + + struct TokenRecord { + std::string text; + uint64_t id; + bool added; + bool special; + }; + std::vector records; + records.reserve(static_cast(count)); + std::unordered_set added_ids; + for (size_t index = 0; index < static_cast(count); ++index) { + uint32_t id = 0; + const uint8_t* text = nullptr; + size_t text_len = 0; + uint8_t is_added = 0; + uint8_t is_special = 0; + if (tokenizers_hf_token_at( + handle, index, &id, &text, &text_len, &is_added, &is_special) != + 0 || + (text == nullptr && text_len != 0)) { + return Error::ParseFailure; + } + records.push_back( + {std::string(reinterpret_cast(text), text_len), + id, + is_added != 0, + is_special != 0}); + if (is_added != 0) { + added_ids.insert(id); + if (is_special != 0) { + special_token_ids_.insert(id); + } + } + } + + std::vector> tokens; + std::vector> added_tokens; + std::vector bos_candidates; + std::vector eos_candidates; + tokens.reserve(records.size()); + added_tokens.reserve(added_ids.size()); + for (auto& record : records) { + if (record.special) { + if (is_bos_token(record.text)) { + bos_candidates.push_back(record.id); + } + if (is_eos_token(record.text)) { + eos_candidates.push_back(record.id); + } + } + if (record.added) { + added_tokens.emplace_back(std::move(record.text), record.id); + } else if (added_ids.count(record.id) == 0) { + tokens.emplace_back(std::move(record.text), record.id); + } + } + auto added_map = TokenMap::create(added_tokens); + if (!added_map.ok()) { + return added_map.error(); + } + added_token_map_.emplace(std::move(*added_map)); + auto token_map = TokenMap::create(tokens); + if (!token_map.ok()) { + return token_map.error(); + } + token_map_.emplace(std::move(*token_map)); + vocab_size_ = + static_cast(token_map_->size() + added_token_map_->size()); + + uint32_t bos = 0; + uint32_t eos = 0; + const auto bos_status = tokenizers_hf_post_token(handle, 0, &bos); + const auto eos_status = tokenizers_hf_post_token(handle, 1, &eos); + if (bos_status < 0 || eos_status < 0) { + return Error::ParseFailure; + } + if (bos_status == 0) { + bos_tok_ = bos; + has_bos_token_ = true; + } + if (eos_status == 0) { + eos_tok_ = eos; + has_eos_token_ = true; + } + if (!has_bos_token_ || !has_eos_token_) { + if (!has_bos_token_ && bos_candidates.size() == 1) { + bos_tok_ = bos_candidates.front(); + has_bos_token_ = true; + } + if (!has_eos_token_ && eos_candidates.size() == 1) { + eos_tok_ = eos_candidates.front(); + has_eos_token_ = true; + } + } + if (!has_bos_token_ || !has_eos_token_) { + return Error::ParseFailure; + } + return Error::Ok; +} + +Result RustHFTokenizer::id_to_piece(uint64_t token) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto piece = token_map_->tryGetString(token)) { + return std::string(*piece); + } + if (auto piece = added_token_map_->tryGetString(token)) { + return std::string(*piece); + } + return Error::OutOfRange; +} + +Result RustHFTokenizer::piece_to_id(const std::string& text) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (auto id = token_map_->tryGetInteger(text)) { + return *id; + } + if (auto id = added_token_map_->tryGetInteger(text)) { + return *id; + } + return Error::OutOfRange; +} + +Result> RustHFTokenizer::encode( + const std::string& input, + int8_t bos, + int8_t eos) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (input.size() > + static_cast(std::numeric_limits::max())) { + return Error::EncodeFailure; + } + if ((bos > 0 && !has_bos_token_) || (eos > 0 && !has_eos_token_)) { + return Error::EncodeFailure; + } + + std::vector output(input.size()); + auto count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + output.data(), + output.size()); + if (count < 0) { + return Error::EncodeFailure; + } + if (static_cast(count) > output.size()) { + output.resize(static_cast(count)); + count = tokenizers_hf_encode( + handle_.get(), + reinterpret_cast(input.data()), + input.size(), + output.data(), + output.size()); + if (count < 0 || static_cast(count) > output.size()) { + return Error::EncodeFailure; + } + } + output.resize(static_cast(count)); + const auto bos_count = bos > 0 ? static_cast(bos) : 0; + const auto eos_count = eos > 0 ? static_cast(eos) : 0; + std::vector tokens; + tokens.reserve(output.size() + bos_count + eos_count); + tokens.insert(tokens.end(), bos_count, bos_tok_); + tokens.insert(tokens.end(), output.begin(), output.end()); + tokens.insert(tokens.end(), eos_count, eos_tok_); + return tokens; +} + +Result RustHFTokenizer::decode( + uint64_t prev_token, + uint64_t token, + bool skip_special_tokens) const { + if (!initialized_) { + return Error::Uninitialized; + } + if (!byte_level_) { + if (token > std::numeric_limits::max() || + prev_token > std::numeric_limits::max()) { + return Error::DecodeFailure; + } + auto count = tokenizers_hf_decode( + handle_.get(), + static_cast(prev_token), + prev_token != 0, + static_cast(token), + skip_special_tokens, + nullptr, + 0); + if (count < 0) { + return Error::DecodeFailure; + } + std::string decoded(static_cast(count), '\0'); + count = tokenizers_hf_decode( + handle_.get(), + static_cast(prev_token), + prev_token != 0, + static_cast(token), + skip_special_tokens, + reinterpret_cast(decoded.data()), + decoded.size()); + if (count < 0 || static_cast(count) > decoded.size()) { + return Error::DecodeFailure; + } + decoded.resize(static_cast(count)); + return decoded; + } + + std::string_view piece; + if (auto regular = token_map_->tryGetString(token)) { + piece = *regular; + } else if (auto added = added_token_map_->tryGetString(token)) { + if (skip_special_tokens && special_token_ids_.count(token) != 0) { + return std::string(); + } + piece = *added; + } else { + return Error::DecodeFailure; + } + + return decode_byte_level(piece); +} + +} // namespace tokenizers diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b9e0a034..6e80b084 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,12 @@ foreach(test_source_file ${test_source_files}) get_filename_component(test_name ${test_source_file} NAME_WE) message(STATUS "Configuring unit test ${test_name}") add_executable(${test_name} ${test_source_file}) + if(test_name STREQUAL "test_rust_hf_tokenizer") + target_sources( + ${test_name} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src/rust_hf_tokenizer.cpp + ) + endif() target_include_directories( ${test_name} PRIVATE GTEST_INCLUDE_PATH ${TOKENIZERS_INCLUDE_DIRS} ) diff --git a/test/test_rust_hf_tokenizer.cpp b/test/test_rust_hf_tokenizer.cpp new file mode 100644 index 00000000..15733d73 --- /dev/null +++ b/test/test_rust_hf_tokenizer.cpp @@ -0,0 +1,213 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +constexpr uint32_t kByteLevelFlag = 1U << 2; +uint32_t g_config_flags = kByteLevelFlag; +bool g_use_known_eos = true; + +struct TokenRecord { + uint32_t id; + std::string text; + bool added; + bool special; +}; + +const std::array kRecords = { + {{1, "", true, true}, + {2, "", true, true}, + {3, "hello", false, false}, + {4, "\xC4\xA0world", false, false}}}; + +class TemporaryTokenizerFile { + public: + TemporaryTokenizerFile() + : path_( + std::filesystem::temp_directory_path() / + "pytorch-tokenizers-rust-adapter-test.json") { + std::ofstream(path_).put('\0'); + } + + ~TemporaryTokenizerFile() { + std::error_code error; + std::filesystem::remove(path_, error); + } + + std::string string() const { + return path_.string(); + } + + private: + std::filesystem::path path_; +}; + +} // namespace + +extern "C" { + +void* tokenizers_hf_create(const char*) { + return new uint8_t(0); +} + +intptr_t tokenizers_hf_encode( + const void*, + const uint8_t*, + size_t, + uint32_t* output, + size_t output_capacity) { + constexpr std::array encoded = {3, 4}; + if (output_capacity < encoded.size()) { + return encoded.size(); + } + std::copy(encoded.begin(), encoded.end(), output); + return encoded.size(); +} + +intptr_t tokenizers_hf_decode( + const void*, + uint32_t, + uint8_t, + uint32_t token, + uint8_t skip_special, + uint8_t* output, + size_t output_capacity) { + const std::string decoded = + skip_special != 0 && token == 2 ? "" : "decoded world"; + if (output_capacity < decoded.size()) { + return decoded.size(); + } + if (!decoded.empty()) { + std::copy(decoded.begin(), decoded.end(), output); + } + return decoded.size(); +} + +intptr_t tokenizers_hf_token_count(const void*) { + return kRecords.size(); +} + +int32_t tokenizers_hf_token_at( + const void*, + size_t index, + uint32_t* id, + const uint8_t** text, + size_t* text_len, + uint8_t* is_added, + uint8_t* is_special) { + if (index >= kRecords.size()) { + return -1; + } + const auto& record = kRecords[index]; + static const std::string kUnknownEos = ""; + const auto& record_text = + index == 1 && !g_use_known_eos ? kUnknownEos : record.text; + *id = record.id; + *text = reinterpret_cast(record_text.data()); + *text_len = record_text.size(); + *is_added = record.added; + *is_special = record.special; + return 0; +} + +int32_t tokenizers_hf_post_token(const void*, uint8_t suffix, uint32_t* token) { + if (suffix != 0) { + return 1; + } + *token = 1; + return 0; +} + +uint32_t tokenizers_hf_config_flags(const void*) { + return g_config_flags; +} + +void tokenizers_hf_destroy(void* handle) { + delete static_cast(handle); +} + +} // extern "C" + +namespace tokenizers { +namespace { + +TEST(RustHFTokenizerTest, PreservesIndependentBosAndEosCounts) { + g_config_flags = kByteLevelFlag; + g_use_known_eos = true; + TemporaryTokenizerFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + EXPECT_EQ(tokenizer.bos_tok(), 1); + EXPECT_EQ(tokenizer.eos_tok(), 2); + + auto tokens = tokenizer.encode("hello world", 2, 1); + ASSERT_TRUE(tokens.ok()); + EXPECT_EQ(*tokens, (std::vector{1, 1, 3, 4, 2})); + + auto bos_only_tokens = tokenizer.encode("hello world", 1, 0); + ASSERT_TRUE(bos_only_tokens.ok()); + EXPECT_EQ(*bos_only_tokens, (std::vector{1, 3, 4})); +} + +TEST(RustHFTokenizerTest, DecodesByteLevelPieces) { + g_config_flags = kByteLevelFlag; + g_use_known_eos = true; + TemporaryTokenizerFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + + auto decoded = tokenizer.decode(3, 4); + ASSERT_TRUE(decoded.ok()); + EXPECT_EQ(*decoded, " world"); + + auto skipped = tokenizer.decode(4, 2, true); + ASSERT_TRUE(skipped.ok()); + EXPECT_TRUE(skipped->empty()); +} + +TEST(RustHFTokenizerTest, UsesRustDecoderForNonByteLevelJson) { + g_config_flags = 0; + g_use_known_eos = true; + TemporaryTokenizerFile file; + RustHFTokenizer tokenizer; + ASSERT_EQ(tokenizer.load(file.string()), Error::Ok); + + auto decoded = tokenizer.decode(3, 4); + ASSERT_TRUE(decoded.ok()); + EXPECT_EQ(*decoded, "decoded world"); + + auto skipped = tokenizer.decode(4, 2, true); + ASSERT_TRUE(skipped.ok()); + EXPECT_TRUE(skipped->empty()); + g_config_flags = kByteLevelFlag; +} + +TEST(RustHFTokenizerTest, RejectsJsonWithoutBosOrEosMetadata) { + TemporaryTokenizerFile file; + RustHFTokenizer tokenizer; + g_config_flags = kByteLevelFlag; + g_use_known_eos = false; + EXPECT_EQ(tokenizer.load(file.string()), Error::ParseFailure); + EXPECT_FALSE(tokenizer.is_loaded()); + g_use_known_eos = true; +} + +} // namespace +} // namespace tokenizers