diff --git a/CMakeLists.txt b/CMakeLists.txt index 46c3cd5c570..4bbbaf2f21e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1002,6 +1002,8 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() + set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER}) + set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) set(CMAKE_POSITION_INDEPENDENT_CODE diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index 17016b955e7..5f4579f5123 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -49,6 +49,12 @@ set(runner_deps executorch_core extension_module extension_tensor extension_llm_sampler extension_memory_allocator tokenizers::tokenizers ) +if(EXECUTORCH_BUILD_HF_RUST_TOKENIZER) + list(APPEND runner_deps tokenizers::hf_rust_tokenizer) + target_compile_definitions( + extension_llm_runner PRIVATE EXECUTORCH_USE_HF_RUST_TOKENIZER=1 + ) +endif() # depend on arange_utils if(NOT TARGET kernels_util_all_deps) diff --git a/extension/llm/runner/llm_runner_helper.cpp b/extension/llm/runner/llm_runner_helper.cpp index 105b3f0c9f4..887039a65da 100644 --- a/extension/llm/runner/llm_runner_helper.cpp +++ b/extension/llm/runner/llm_runner_helper.cpp @@ -25,6 +25,9 @@ #include #include #include +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER +#include +#endif namespace executorch::extension::llm { @@ -50,6 +53,13 @@ std::unique_ptr load_tokenizer( return tekken_tokenizer; } } +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER + auto rust_tok_tokenizer = std::make_unique(); + if (rust_tok_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded Hugging Face Rust tokenizer"); + return rust_tok_tokenizer; + } +#endif auto json_tokenizer = std::make_unique(); if (json_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { ET_LOG(Info, "Loaded json tokenizer"); diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3fc1df71045..e5c2469a3bb 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3fc1df71045f76841ee912b31f1f176067de2e16 +Subproject commit e5c2469a3bbecbcf884a2ae4366a3dbecbdc7fc0 diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 89e679aec7c..2577bbcbfc0 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -100,6 +100,10 @@ define_overridable_option( define_overridable_option( EXECUTORCH_BUILD_EXTENSION_LLM "Build the LLM extension" BOOL OFF ) +define_overridable_option( + EXECUTORCH_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face Rust tokenizer backend" BOOL OFF +) define_overridable_option( EXECUTORCH_BUILD_EXTENSION_ETDUMP_APPLE "Build the ETDump Apple extension" BOOL OFF