diff --git a/CMakeLists.txt b/CMakeLists.txt index 46c3cd5c570..90139bc1c9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1002,6 +1002,9 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() + set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER}) + set(TOKENIZERS_HF_RUST_FORMATS ${EXECUTORCH_HF_RUST_FORMATS}) + set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) set(CMAKE_POSITION_INDEPENDENT_CODE diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index 17016b955e7..5f4579f5123 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -49,6 +49,12 @@ set(runner_deps executorch_core extension_module extension_tensor extension_llm_sampler extension_memory_allocator tokenizers::tokenizers ) +if(EXECUTORCH_BUILD_HF_RUST_TOKENIZER) + list(APPEND runner_deps tokenizers::hf_rust_tokenizer) + target_compile_definitions( + extension_llm_runner PRIVATE EXECUTORCH_USE_HF_RUST_TOKENIZER=1 + ) +endif() # depend on arange_utils if(NOT TARGET kernels_util_all_deps) diff --git a/extension/llm/runner/llm_runner_helper.cpp b/extension/llm/runner/llm_runner_helper.cpp index 105b3f0c9f4..887039a65da 100644 --- a/extension/llm/runner/llm_runner_helper.cpp +++ b/extension/llm/runner/llm_runner_helper.cpp @@ -25,6 +25,9 @@ #include #include #include +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER +#include +#endif namespace executorch::extension::llm { @@ -50,6 +53,13 @@ std::unique_ptr load_tokenizer( return tekken_tokenizer; } } +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER + auto rust_tok_tokenizer = std::make_unique(); + if (rust_tok_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded Hugging Face Rust tokenizer"); + return rust_tok_tokenizer; + } +#endif auto json_tokenizer = std::make_unique(); if (json_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { ET_LOG(Info, "Loaded json tokenizer"); diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3fc1df71045..48970bb5640 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3fc1df71045f76841ee912b31f1f176067de2e16 +Subproject commit 48970bb564014e075a0c05cda787cebcb3bbc1fd diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 89e679aec7c..a63e48c8da0 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -100,6 +100,17 @@ define_overridable_option( define_overridable_option( EXECUTORCH_BUILD_EXTENSION_LLM "Build the LLM extension" BOOL OFF ) +define_overridable_option( + EXECUTORCH_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face Rust tokenizer backend" BOOL OFF +) +set(EXECUTORCH_HF_RUST_FORMATS + "json" + CACHE STRING "Hugging Face Rust tokenizer formats: json, tok, or json,tok" +) +set_property( + CACHE EXECUTORCH_HF_RUST_FORMATS PROPERTY STRINGS json tok "json,tok" +) define_overridable_option( EXECUTORCH_BUILD_EXTENSION_ETDUMP_APPLE "Build the ETDump Apple extension" BOOL OFF