From 303182adf55f6ea27658d2121710bc3e85a98810 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 3 Sep 2026 18:56:52 -0700 Subject: [PATCH 1/5] Add opt-in Hugging Face .tok tokenizer integration --- CMakeLists.txt | 4 ++++ extension/llm/runner/CMakeLists.txt | 6 ++++++ extension/llm/runner/llm_runner_helper.cpp | 10 ++++++++++ extension/llm/tokenizers | 2 +- tools/cmake/preset/default.cmake | 4 ++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46c3cd5c570..387279ebaa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1002,6 +1002,10 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() + set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER + ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER} + ) + set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) set(CMAKE_POSITION_INDEPENDENT_CODE diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index 17016b955e7..5f4579f5123 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -49,6 +49,12 @@ set(runner_deps executorch_core extension_module extension_tensor extension_llm_sampler extension_memory_allocator tokenizers::tokenizers ) +if(EXECUTORCH_BUILD_HF_RUST_TOKENIZER) + list(APPEND runner_deps tokenizers::hf_rust_tokenizer) + target_compile_definitions( + extension_llm_runner PRIVATE EXECUTORCH_USE_HF_RUST_TOKENIZER=1 + ) +endif() # depend on arange_utils if(NOT TARGET kernels_util_all_deps) diff --git a/extension/llm/runner/llm_runner_helper.cpp b/extension/llm/runner/llm_runner_helper.cpp index 105b3f0c9f4..1d895751cd7 100644 --- a/extension/llm/runner/llm_runner_helper.cpp +++ b/extension/llm/runner/llm_runner_helper.cpp @@ -25,6 +25,9 @@ #include #include #include +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER +#include +#endif namespace executorch::extension::llm { @@ -50,6 +53,13 @@ std::unique_ptr load_tokenizer( return tekken_tokenizer; } } +#ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER + auto rust_tok_tokenizer = std::make_unique(); + if (rust_tok_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded Hugging Face .tok tokenizer"); + return rust_tok_tokenizer; + } +#endif auto json_tokenizer = std::make_unique(); if (json_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { ET_LOG(Info, "Loaded json tokenizer"); diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3fc1df71045..c250cc9bc41 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3fc1df71045f76841ee912b31f1f176067de2e16 +Subproject commit c250cc9bc41f8e3986b9b0165fb75eea40ae3a5d diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 89e679aec7c..fc85e54e57b 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -100,6 +100,10 @@ define_overridable_option( define_overridable_option( EXECUTORCH_BUILD_EXTENSION_LLM "Build the LLM extension" BOOL OFF ) +define_overridable_option( + EXECUTORCH_BUILD_HF_RUST_TOKENIZER + "Build the opt-in Hugging Face .tok tokenizer backend" BOOL OFF +) define_overridable_option( EXECUTORCH_BUILD_EXTENSION_ETDUMP_APPLE "Build the ETDump Apple extension" BOOL OFF From f38fa5c7362630be77e1122c76df1f61547c5445 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 11:27:36 -0700 Subject: [PATCH 2/5] Update Hugging Face tokenizer integration Point at the rebased tokenizer dependency with corrected BOS/EOS semantics, safe decoder gating, and regression coverage. Apply the required CMake formatting. AI-assisted-by: Codex --- CMakeLists.txt | 4 +--- extension/llm/tokenizers | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 387279ebaa2..4bbbaf2f21e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1002,9 +1002,7 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() - set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER - ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER} - ) + set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER}) set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index c250cc9bc41..16692379717 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit c250cc9bc41f8e3986b9b0165fb75eea40ae3a5d +Subproject commit 166923797174d706019ef10116fd2a457a964dcd From 5e3887ac07d4b0cdeae16b7e66ac00c6a9d574e9 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 13:26:50 -0700 Subject: [PATCH 3/5] Support Hugging Face JSON and tok backends AI-assisted-by: Codex --- CMakeLists.txt | 1 + extension/llm/runner/llm_runner_helper.cpp | 2 +- extension/llm/tokenizers | 2 +- tools/cmake/preset/default.cmake | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bbbaf2f21e..90139bc1c9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1003,6 +1003,7 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER}) + set(TOKENIZERS_HF_RUST_FORMATS ${EXECUTORCH_HF_RUST_FORMATS}) set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) diff --git a/extension/llm/runner/llm_runner_helper.cpp b/extension/llm/runner/llm_runner_helper.cpp index 1d895751cd7..887039a65da 100644 --- a/extension/llm/runner/llm_runner_helper.cpp +++ b/extension/llm/runner/llm_runner_helper.cpp @@ -56,7 +56,7 @@ std::unique_ptr load_tokenizer( #ifdef EXECUTORCH_USE_HF_RUST_TOKENIZER auto rust_tok_tokenizer = std::make_unique(); if (rust_tok_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { - ET_LOG(Info, "Loaded Hugging Face .tok tokenizer"); + ET_LOG(Info, "Loaded Hugging Face Rust tokenizer"); return rust_tok_tokenizer; } #endif diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 16692379717..5c68507a043 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 166923797174d706019ef10116fd2a457a964dcd +Subproject commit 5c68507a043ea2d56a0166109a4bb1a79d1a69fd diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index fc85e54e57b..a63e48c8da0 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -102,7 +102,14 @@ define_overridable_option( ) define_overridable_option( EXECUTORCH_BUILD_HF_RUST_TOKENIZER - "Build the opt-in Hugging Face .tok tokenizer backend" BOOL OFF + "Build the opt-in Hugging Face Rust tokenizer backend" BOOL OFF +) +set(EXECUTORCH_HF_RUST_FORMATS + "json" + CACHE STRING "Hugging Face Rust tokenizer formats: json, tok, or json,tok" +) +set_property( + CACHE EXECUTORCH_HF_RUST_FORMATS PROPERTY STRINGS json tok "json,tok" ) define_overridable_option( EXECUTORCH_BUILD_EXTENSION_ETDUMP_APPLE "Build the ETDump Apple extension" From 25dddf4194c2f86b3e7aea756c2e9f9d5595cd72 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 13:45:50 -0700 Subject: [PATCH 4/5] Update tokenizer size measurements Generated with assistance from Codex. --- extension/llm/tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 5c68507a043..48970bb5640 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 5c68507a043ea2d56a0166109a4bb1a79d1a69fd +Subproject commit 48970bb564014e075a0c05cda787cebcb3bbc1fd From 1e71054a8e80c63d6d79075d0ca3d578e303792f Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 24 Sep 2026 16:46:15 -0700 Subject: [PATCH 5/5] Make Hugging Face integration JSON-only Generated with assistance from Codex. --- CMakeLists.txt | 1 - extension/llm/tokenizers | 2 +- tools/cmake/preset/default.cmake | 7 ------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90139bc1c9c..4bbbaf2f21e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1003,7 +1003,6 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() set(TOKENIZERS_BUILD_HF_RUST_TOKENIZER ${EXECUTORCH_BUILD_HF_RUST_TOKENIZER}) - set(TOKENIZERS_HF_RUST_FORMATS ${EXECUTORCH_HF_RUST_FORMATS}) set(TOKENIZERS_OPTIMIZE_SIZE ${EXECUTORCH_OPTIMIZE_SIZE}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 48970bb5640..e5c2469a3bb 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 48970bb564014e075a0c05cda787cebcb3bbc1fd +Subproject commit e5c2469a3bbecbcf884a2ae4366a3dbecbdc7fc0 diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index a63e48c8da0..2577bbcbfc0 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -104,13 +104,6 @@ define_overridable_option( EXECUTORCH_BUILD_HF_RUST_TOKENIZER "Build the opt-in Hugging Face Rust tokenizer backend" BOOL OFF ) -set(EXECUTORCH_HF_RUST_FORMATS - "json" - CACHE STRING "Hugging Face Rust tokenizer formats: json, tok, or json,tok" -) -set_property( - CACHE EXECUTORCH_HF_RUST_FORMATS PROPERTY STRINGS json tok "json,tok" -) define_overridable_option( EXECUTORCH_BUILD_EXTENSION_ETDUMP_APPLE "Build the ETDump Apple extension" BOOL OFF