From cea2c215530c41267ae41958075fcecc253f95ff Mon Sep 17 00:00:00 2001 From: "Gurmukhani, Anjul" Date: Mon, 26 Oct 2026 22:34:16 -0700 Subject: [PATCH 01/13] feat(core): run Llama 3.2 1B/3B and 3.1 8B on corelib's llama model - Add a llama3 core backend over corelib's built-in llama model, catalogued as llama3.2-core:1b/3b and llama3.1-core:8b on dev04. - Align the Llama3 frontend's capacity check, poison guard and failure handling with Phi-4/Qwen3. - _shared_generate: leave a token cut off by the budget, the decode window or a cancel out of token_history, so the next turn prefills it. - Tests: test/llama_core frontend suite, test_llama_core_backend, and cut-off-token cases for the core and flm backends. --- src/common/AutoModel/automodel.cpp | 7 + src/common/AutoModel/builtin_backends.cpp | 4 + src/common/AutoModel/modeling_llama3.cpp | 45 ++- .../llama/core/dev04/llama_core_backend.cpp | 22 ++ src/include/AutoModel/modeling_llama3.hpp | 6 + .../llama/core/dev04/llama_core_backend.hpp | 28 ++ src/model_info.json | 78 ++++ src/model_list.json | 123 +++++- src/test/core_model/CMakeLists.txt | 3 + .../core_model/test_llama_core_backend.cpp | 105 +++++ src/test/llama_core/CMakeLists.txt | 45 +++ src/test/llama_core/test_llama_frontend.cpp | 362 ++++++++++++++++++ src/test/phi4_core/test_phi4_frontend.cpp | 46 +++ 13 files changed, 868 insertions(+), 6 deletions(-) create mode 100644 src/common/models/llama/core/dev04/llama_core_backend.cpp create mode 100644 src/include/models/llama/core/dev04/llama_core_backend.hpp create mode 100644 src/test/core_model/test_llama_core_backend.cpp create mode 100644 src/test/llama_core/CMakeLists.txt create mode 100644 src/test/llama_core/test_llama_frontend.cpp diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 4d6cb76e2..d88d2e2b0 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -808,6 +808,8 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ header_print("WARNING", "Max length reached, stopping generation..."); reason = MAX_LENGTH_REACHED; meta_info.stop_reason = reason; + // Sampled from the prefill, so no engine has run it. + this->token_history.pop_back(); return result; } while (true){ @@ -862,6 +864,11 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ break; } } + if (reason != EOT_DETECTED) { + // Stopped on the budget, the window or a cancel before the token + // sampled last was run, so the next prompt has to carry it. + this->token_history.pop_back(); + } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; if (this->total_tokens >= decode_cap){ diff --git a/src/common/AutoModel/builtin_backends.cpp b/src/common/AutoModel/builtin_backends.cpp index 5dfbcb2eb..bdb17f1f3 100644 --- a/src/common/AutoModel/builtin_backends.cpp +++ b/src/common/AutoModel/builtin_backends.cpp @@ -10,6 +10,7 @@ #if defined(FLM_ENABLE_CORE) #include "models/gemma4/core/dev04/gemma4_core_backend.hpp" +#include "models/llama/core/dev04/llama_core_backend.hpp" #include "models/phi4/core/dev04/phi4_core_backend.hpp" #include "models/qwen3/core/dev04/qwen3_core_backend.hpp" #include "models/qwen35/core/dev04/qwen35_core_backend.hpp" @@ -74,6 +75,9 @@ void register_builtin_backends(BackendRegistry& registry) { registry.register_backend("qwen3.6-moe", flm::backend::kCoreBackendId, flm::qwen36::core_factory(), flm::qwen36::core_traits()); + registry.register_backend("llama3", flm::backend::kCoreBackendId, + flm::llama::core_factory(), + flm::llama::core_traits()); #endif } diff --git a/src/common/AutoModel/modeling_llama3.cpp b/src/common/AutoModel/modeling_llama3.cpp index f187fa973..12b314a68 100644 --- a/src/common/AutoModel/modeling_llama3.cpp +++ b/src/common/AutoModel/modeling_llama3.cpp @@ -40,7 +40,18 @@ std::string Llama3::apply_chat_template(nlohmann::ordered_json& messages, nlohma return this->chat_tmpl->apply(inputs); } +void Llama3::fail_inference(const std::string& detail) { + const bool poisoned = this->backend_ && this->backend_->poisoned(); + this->_shared_after_inference_failure(poisoned); + std::string message = poisoned + ? "Inference failed; unload/reload is required because the model is poisoned" + : "Inference failed; the current conversation was cleared"; + if (!detail.empty()) message += " (" + detail + ")"; + throw ModelRequestError(500, true, message); +} + bool Llama3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { + this->_shared_guard_poisoned(); // preprocess this->profiler_list[TKOEN_ENCODE_TIME].start(); std::string templated_text; @@ -66,23 +77,47 @@ bool Llama3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std: // The template's generation prompt ("<|start_header_id|>assistant<|end_header_id|>\n\n") is how the next turn // renders this reply's header, so the whole prompt can be saved: nothing is left out of the prefill (trim 0). if (this->_prefix_slot_mode(meta_info, tokens.size(), 0)) { - return this->_prefix_text_insert(meta_info, tokens, 0, "<|start_header_id|>user", is_cancelled); + try { + return this->_prefix_text_insert(meta_info, tokens, 0, "<|start_header_id|>user", is_cancelled, input.requested_max_new_tokens); + } catch (const ModelRequestError&) { + throw; + } catch (const std::exception& error) { + this->fail_inference(error.what()); + } catch (...) { + this->fail_inference(); + } } this->_prefix_legacy_path(); - return this->_shared_insert(meta_info, tokens, is_cancelled); + try { + return this->_shared_insert(meta_info, tokens, is_cancelled, nullptr, 0, input.requested_max_new_tokens); + } catch (const ModelRequestError&) { + throw; + } catch (const std::exception& error) { + this->fail_inference(error.what()); + } catch (...) { + this->fail_inference(); + } } std::string Llama3::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { - //header_print("is_cancelled", is_cancelled); - return this->_shared_generate(meta_info, length_limit, os, is_cancelled); + this->_shared_guard_poisoned(); + try { + return this->_shared_generate(meta_info, length_limit, os, is_cancelled); + } catch (const ModelRequestError&) { + throw; + } catch (const std::exception& error) { + this->fail_inference(error.what()); + } catch (...) { + this->fail_inference(); + } } std::string Llama3::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { if (!this->insert(meta_info, input)) { return ""; } - return this->_shared_generate(meta_info, length_limit, os); + return this->generate(meta_info, length_limit, os); } /************ DeepSeek_r1_8b family **************/ diff --git a/src/common/models/llama/core/dev04/llama_core_backend.cpp b/src/common/models/llama/core/dev04/llama_core_backend.cpp new file mode 100644 index 000000000..4c84767ee --- /dev/null +++ b/src/common/models/llama/core/dev04/llama_core_backend.cpp @@ -0,0 +1,22 @@ +/// \file llama_core_backend.cpp +/// \brief Llama 3.2 1B and 3B, and Llama 3.1 8B, on corelib's "llama" model +#include "models/llama/core/dev04/llama_core_backend.hpp" + +#include "core/corelib_model_backend.hpp" + +namespace flm::llama { + +/// \note A multi-row pass starts only at position 0, so a later prompt re-runs +/// the conversation with it. +flm::backend::BackendFactory core_factory() { + return core::Factory({ + .model = {"llama", "Llama", kCoreContextLimit, core::Continuation::kFromStart, + core::Rewind::kSeek}, + .context_rule = "1..4096", + // <|eot_id|>, <|eom_id|>, <|end_of_text|>: eos_token_id in the + // catalog's config.json + .eos_ids = {128009, 128008, 128001}, + }); +} + +} // namespace flm::llama diff --git a/src/include/AutoModel/modeling_llama3.hpp b/src/include/AutoModel/modeling_llama3.hpp index dfb19dbba..8ba5c1884 100644 --- a/src/include/AutoModel/modeling_llama3.hpp +++ b/src/include/AutoModel/modeling_llama3.hpp @@ -12,6 +12,12 @@ class Llama3 : public AutoModel { private: void setup_tokenizer(std::string model_path); + /// \brief clear the conversation after a failed inference and report it + /// \param detail the backend's own message, when the failure carried one + /// \throws ModelRequestError 500, asking for a reload when the backend is poisoned + /// \note The detail is appended to the thrown message. Without it the + /// corelib label that names the failing op is lost at the catch. + [[noreturn]] void fail_inference(const std::string& detail = {}); public: Llama3(flm_rt::device* npu_device_inst); diff --git a/src/include/models/llama/core/dev04/llama_core_backend.hpp b/src/include/models/llama/core/dev04/llama_core_backend.hpp new file mode 100644 index 000000000..d132089d0 --- /dev/null +++ b/src/include/models/llama/core/dev04/llama_core_backend.hpp @@ -0,0 +1,28 @@ +/// \file llama_core_backend.hpp +/// \brief The ryzenai-corelib backend for Llama 3.2 1B and 3B, and Llama 3.1 8B +/// \note Compiled only when FLM_ENABLE_CORE is on. +#pragma once + +#include "AutoModel/model_backend.hpp" + +#include + +namespace flm::llama { + +/// \brief the largest context this backend can hold +inline constexpr std::uint32_t kCoreContextLimit = 4096; + +/// \brief what the frontend must know before building this backend +/// \note Inline so the registry can reject a request before the engine exists. +inline flm::backend::BackendTraits core_traits() { + flm::backend::BackendTraits traits; + traits.needs_npu_xclbin = false; + traits.supports_preemption = false; + traits.max_context_length = kCoreContextLimit; + return traits; +} + +/// \brief a factory building the core backend +flm::backend::BackendFactory core_factory(); + +} // namespace flm::llama diff --git a/src/model_info.json b/src/model_info.json index a24e965f8..1d3f2c9bc 100644 --- a/src/model_info.json +++ b/src/model_info.json @@ -4778,5 +4778,83 @@ "xetHash": "56a94fd74fdd847a3069ba83ddea8801feea984490eda49d15e9e3c529266bb6", "path": "vision_weight.q4nx" } + ], + "llama3.2-core:1b": [ + { + "type": "file", + "path": "Llama-3.2-1B-Instruct-Q8_0.gguf", + "size": 1321083008, + "sha256": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 9085657, + "sha256": "79e3e522635f3171300913bb421464a87de6222182a0570b9b2ccba2a964b2b4" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 4331, + "sha256": "01b0d6dd3c391ffc2facc01592ac07f25e8c4212d3450b8103042e8a5fcbafa8" + }, + { + "type": "file", + "path": "config.json", + "size": 1019, + "sha256": "a53d38c545333679d523863f60a45f04166e52d15793864da3464712810f8bd9" + } + ], + "llama3.2-core:3b": [ + { + "type": "file", + "path": "Llama-3.2-3B-Instruct-Q8_0.gguf", + "size": 3421899296, + "sha256": "b5607b5090a8280063fff2d706bb3408ca6542341b06aab39c3eca0a28575921" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 9085657, + "sha256": "79e3e522635f3171300913bb421464a87de6222182a0570b9b2ccba2a964b2b4" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 4331, + "sha256": "01b0d6dd3c391ffc2facc01592ac07f25e8c4212d3450b8103042e8a5fcbafa8" + }, + { + "type": "file", + "path": "config.json", + "size": 1019, + "sha256": "f83c8c4bfd00c7e04c6a4b8d9ed471193bed0aa0304c32dff847a36586f8291e" + } + ], + "llama3.1-core:8b": [ + { + "type": "file", + "path": "Meta-Llama-3.1-8B-Instruct-Q8_0.gguf", + "size": 8540775840, + "sha256": "9da71c45c90a821809821244d4971e5e5dfad7eb091f0b8ff0546392393b6283" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 9085657, + "sha256": "79e3e522635f3171300913bb421464a87de6222182a0570b9b2ccba2a964b2b4" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 4331, + "sha256": "01b0d6dd3c391ffc2facc01592ac07f25e8c4212d3450b8103042e8a5fcbafa8" + }, + { + "type": "file", + "path": "config.json", + "size": 1020, + "sha256": "82ff79502694a38eacd424763c5f3cc8a95843dd233d830b74dee5b3a14dde9b" + } ] } diff --git a/src/model_list.json b/src/model_list.json index d7d8a1c1e..182e671ad 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -2167,6 +2167,127 @@ "footprint": 8.71, "model_info_key": "qwen3-core:8b" } - } + }, + "llama3.2-core": { + "1b": { + "supported_platforms": ["dev04"], + "name": "llama3.2-1b-core", + "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/067b946cf014b7c697f3654f621d577a3e3afd1c", + "file_url": "https://huggingface.co/api/models/bartowski/Llama-3.2-1B-Instruct-GGUF/tree/067b946cf014b7c697f3654f621d577a3e3afd1c", + "size": 1330174015, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "llama3", + "think": false, + "think_toggleable": false, + "parameter_size": "1B", + "quantization_level": "Q8_0 -> group-64" + }, + "flm_min_version": "1.0.6", + "vlm": false, + "files": [ + "Llama-3.2-1B-Instruct-Q8_0.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json" + ], + "file_sources": { + "tokenizer.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-1B-NPU2", + "revision": "7bb88fcd5653534fe065b911123d10294b62a8d1" + }, + "tokenizer_config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-1B-NPU2", + "revision": "7bb88fcd5653534fe065b911123d10294b62a8d1" + }, + "config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-1B-NPU2", + "revision": "7bb88fcd5653534fe065b911123d10294b62a8d1" + } + }, + "footprint": 1.32 + }, + "3b": { + "supported_platforms": ["dev04"], + "name": "llama3.2-3b-core", + "url": "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/5ab33fa94d1d04e903623ae72c95d1696f09f9e8", + "file_url": "https://huggingface.co/api/models/bartowski/Llama-3.2-3B-Instruct-GGUF/tree/5ab33fa94d1d04e903623ae72c95d1696f09f9e8", + "size": 3430990303, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "llama3", + "think": false, + "think_toggleable": false, + "parameter_size": "3B", + "quantization_level": "Q8_0 -> group-64" + }, + "flm_min_version": "1.0.6", + "vlm": false, + "files": [ + "Llama-3.2-3B-Instruct-Q8_0.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json" + ], + "file_sources": { + "tokenizer.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-3B-NPU2", + "revision": "2003cdb18a1ecdda338ff4ff39d3e403476e49ae" + }, + "tokenizer_config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-3B-NPU2", + "revision": "2003cdb18a1ecdda338ff4ff39d3e403476e49ae" + }, + "config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-3B-NPU2", + "revision": "2003cdb18a1ecdda338ff4ff39d3e403476e49ae" + } + }, + "footprint": 3.42 + } + }, + "llama3.1-core": { + "8b": { + "supported_platforms": ["dev04"], + "name": "llama3.1-8b-core", + "url": "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/bf5b95e96dac0462e2a09145ec66cae9a3f12067", + "file_url": "https://huggingface.co/api/models/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/tree/bf5b95e96dac0462e2a09145ec66cae9a3f12067", + "size": 8549866848, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "llama3", + "think": false, + "think_toggleable": false, + "parameter_size": "8B", + "quantization_level": "Q8_0 -> group-64" + }, + "flm_min_version": "1.0.6", + "vlm": false, + "files": [ + "Meta-Llama-3.1-8B-Instruct-Q8_0.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json" + ], + "file_sources": { + "tokenizer.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.1-8B-NPU2", + "revision": "527006483dca6c9d75474ff92b8a6bdd8f257914" + }, + "tokenizer_config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.1-8B-NPU2", + "revision": "527006483dca6c9d75474ff92b8a6bdd8f257914" + }, + "config.json": { + "url": "https://huggingface.co/FastFlowLM/Llama-3.1-8B-NPU2", + "revision": "527006483dca6c9d75474ff92b8a6bdd8f257914" + } + }, + "footprint": 8.54 + } + } } } diff --git a/src/test/core_model/CMakeLists.txt b/src/test/core_model/CMakeLists.txt index 028d235ac..d8750c07a 100644 --- a/src/test/core_model/CMakeLists.txt +++ b/src/test/core_model/CMakeLists.txt @@ -50,3 +50,6 @@ add_core_model_test(test_corelib_model_backend add_core_model_test(test_phi4_core_backend "${FLM_SOURCE_DIR}/common/core/corelib_model_backend.cpp" "${FLM_SOURCE_DIR}/common/models/phi4/core/dev04/phi4_core_backend.cpp") +add_core_model_test(test_llama_core_backend + "${FLM_SOURCE_DIR}/common/core/corelib_model_backend.cpp" + "${FLM_SOURCE_DIR}/common/models/llama/core/dev04/llama_core_backend.cpp") diff --git a/src/test/core_model/test_llama_core_backend.cpp b/src/test/core_model/test_llama_core_backend.cpp new file mode 100644 index 000000000..e5c718cd3 --- /dev/null +++ b/src/test/core_model/test_llama_core_backend.cpp @@ -0,0 +1,105 @@ +/// \file test_llama_core_backend.cpp +/// \brief flm::llama::core_factory against the fake corelib: the spec Llama 3 states +#include "models/llama/core/dev04/llama_core_backend.hpp" +#include "core/corelib_runtime.hpp" +#include "../phi4_core/fake_corelib.hpp" +#include "../phi4_core/test_support.hpp" + +#include +#include +#include +#include +#include +#include + +/// \brief stands in for the one in utils.cpp, which drags in the NPU runtime +/// \note Never consulted: each test installs the fake process runtime first. +namespace utils { +std::string get_executable_directory() { return std::filesystem::current_path().string(); } +} // namespace utils + +namespace { +namespace fs = std::filesystem; +using flm::corelib::CorelibApi; +using flm::corelib::CorelibRuntime; + +constexpr const char* kGguf = "Llama-3.2-1B-Instruct-Q8_0.gguf"; + +/// \brief a Llama 3 package directory holding only the GGUF, removed on the way out +/// \note No tokenizer files: a spec that states its stop ids must not read them. +struct Package { + fs::path root = fs::temp_directory_path() / + ("flm-llama-core-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + + Package() { + fs::create_directories(root); + std::ofstream(root / kGguf) << "x"; + } + ~Package() { + std::error_code ignored; + fs::remove_all(root, ignored); + } +}; + +struct Runtime { + std::shared_ptr runtime; + + Runtime() { + fake_corelib::Reset(); + fake_corelib::GetState().model_vocab = 128256; + runtime = CorelibRuntime::CreateForTest(CorelibApi::ResolveForTest(fake_corelib::Resolver())); + } + ~Runtime() { + try { + runtime.reset(); + CorelibRuntime::ShutdownProcess(); + } catch (...) { + } + } +}; + +flm::backend::BackendContext Context(const Package& package, std::uint32_t length) { + flm::backend::BackendContext context; + context.model_path = package.root.string(); + context.model_info = {{"files", {kGguf, "tokenizer.json", "tokenizer_config.json", "config.json"}}}; + context.context_length = length; + return context; +} + +void TestTheCatalogsGgufRunsAsCorelibsLlama() { + Runtime r; + Package p; + auto backend = flm::llama::core_factory()(Context(p, flm::llama::kCoreContextLimit)); + const auto& state = fake_corelib::GetState(); + TEST_REQUIRE(state.model_names.front() == "llama"); + TEST_REQUIRE(state.model_weights_path == (p.root / kGguf).string()); + TEST_REQUIRE(state.model_max_sequence_length == 4096); + TEST_REQUIRE(backend->id() == flm::backend::kCoreBackendId); + TEST_REQUIRE(backend->max_decode_length() == 4095); + TEST_REQUIRE(!backend->supports_preemption()); + TEST_REQUIRE(!backend->forwards_past_eos()); + // <|eot_id|>, <|eom_id|>, <|end_of_text|> + TEST_REQUIRE((backend->forced_eos_ids() == std::vector{128009, 128008, 128001})); + TEST_REQUIRE(!backend->forced_bos_id().has_value()); +} + +void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { + Runtime r; + Package p; + RequireContains(RequireThrows( + [&] { (void)flm::llama::core_factory()(Context(p, 4097)); }), + "1..4096"); + TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_model_weights_create_path"] == 0); +} + +} // namespace + +int main() { +#define RUN(name) RunTest(&name, #name) + RUN(TestTheCatalogsGgufRunsAsCorelibsLlama); + RUN(TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened); +#undef RUN + std::cout << "All Llama core backend tests passed\n"; + return 0; +} diff --git a/src/test/llama_core/CMakeLists.txt b/src/test/llama_core/CMakeLists.txt new file mode 100644 index 000000000..86152e672 --- /dev/null +++ b/src/test/llama_core/CMakeLists.txt @@ -0,0 +1,45 @@ +cmake_minimum_required(VERSION 3.22) +project(llama_core_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT WIN32) + message(FATAL_ERROR "Llama core tests currently require Windows") +endif() + +# The frontend never includes the corelib header, so only XRT and Boost are +# needed: xrt/xrt_bo.h comes in through buffer.hpp, and boost/any.hpp through it. +find_path(BOOST_INCLUDE_DIR NAMES boost/any.hpp + HINTS "$ENV{CONDA_PREFIX}/Library/include" + "$ENV{USERPROFILE}/anaconda3/Library/include" REQUIRED) +find_path(XRT_INCLUDE_DIR NAMES xrt/xrt_bo.h + HINTS "$ENV{XRT_INCLUDE_DIR}" + "${CMAKE_CURRENT_LIST_DIR}/../../../../xrt_package/xrt/include" + "C:/dev/XRT/src/runtime_src/core/include" REQUIRED) +set(FLM_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../..") + +# builtin_backends.cpp is left out because it names every prebuilt engine +# library; test_llama_frontend.cpp supplies its own empty +# register_builtin_backends and registers a stub instead. +add_executable(test_llama_frontend + test_llama_frontend.cpp + "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" + "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" + "${FLM_SOURCE_DIR}/common/AutoModel/modeling_llama3.cpp") +target_include_directories(test_llama_frontend PRIVATE + "${CMAKE_CURRENT_LIST_DIR}" + "${FLM_SOURCE_DIR}/include" + "${FLM_SOURCE_DIR}/test/phi4_core" + "${BOOST_INCLUDE_DIR}" + "${XRT_INCLUDE_DIR}") +target_compile_definitions(test_llama_frontend PRIVATE + DEV_BUILD=1 __WINDOWS__ USEAVX2=1 DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE WIN32_LEAN_AND_MEAN NOMINMAX) +target_compile_options(test_llama_frontend PRIVATE + $<$:/wd4005 /wd4244>) +target_link_directories(test_llama_frontend PRIVATE "${XRT_INCLUDE_DIR}/../lib") +target_link_libraries(test_llama_frontend PRIVATE xrt_coreutil) + +include(CTest) +add_test(NAME test_llama_frontend COMMAND test_llama_frontend) diff --git a/src/test/llama_core/test_llama_frontend.cpp b/src/test/llama_core/test_llama_frontend.cpp new file mode 100644 index 000000000..d7ddb580e --- /dev/null +++ b/src/test/llama_core/test_llama_frontend.cpp @@ -0,0 +1,362 @@ +/// \file test_llama_frontend.cpp +/// \brief the Llama3 frontend over a stub of the core backend: the budget check, +/// the 500 that clears the conversation, the poisoned engine, and the +/// stop ids the backend states +#include "test_support.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// \brief the builtin set, emptied +/// \note The real one lives in builtin_backends.cpp and names every engine, and +/// with them the prebuilt libraries this suite deliberately does not link. +/// Each test registers the backend it needs instead. +namespace flm::backend { +void register_builtin_backends(BackendRegistry&) {} +} // namespace flm::backend + +namespace { + +/// \brief what the pinned config.json and tokenizer_config.json list as eos_token_id +const std::vector kPackageEosIds = {128001, 128008, 128009}; +/// \brief what llama_core_backend.cpp states +const std::vector kCoreEosIds = {128009, 128008, 128001}; +constexpr int kBosId = 128000; + +std::vector g_encoded_tokens; +std::vector g_samples; +std::size_t g_sample_index{}; + +class FakeEngine final : public causal_lm { +public: + explicit FakeEngine(std::uint32_t limit) : max_length(limit) {} + + buffer forward(int token) override { + ++forward_calls; + forwarded.push_back(token); + if (fail_forward) Fail(); + ++position; + return buffer(1); + } + buffer prefill(std::vector& tokens, void*) override { + ++prefill_calls; + prefill_sizes.push_back(static_cast(tokens.size())); + if (fail_prefill) Fail(); + position += static_cast(tokens.size()); + return buffer(1); + } + void set_context_length(int value) override { position = value; } + void load_weights(Q4NX&) override {} + void update_max_length(std::uint32_t value) override { max_length = value; } + void clear_context() override { + if (poisoned_state) throw std::runtime_error("poisoned"); + ++clear_calls; + position = 0; + } + buffer get_k_cache(int, int) override { return buffer(1); } + buffer get_v_cache(int, int) override { return buffer(1); } + int get_current_context_length() override { return position; } + int checkpoint() override { return position; } + int restore() override { return position; } + + std::uint32_t max_length; + int position{}; + int prefill_calls{}; + int forward_calls{}; + int clear_calls{}; + std::vector prefill_sizes; + std::vector forwarded; + bool fail_prefill{}; + bool fail_forward{}; + /// \brief whether a failure leaves the engine needing a reload, as a + /// corelib failure after submit does + bool poison_on_failure{true}; + bool poisoned_state{}; + +private: + void Fail() { + if (poison_on_failure) poisoned_state = true; + throw std::runtime_error("submitted inference failed"); + } +}; + +struct FactoryState { + int calls{}; + FakeEngine* engine{}; +} g_factory; + +/// \brief the core backend's policy around FakeEngine +/// \note Mirrors CorelibBackend in corelib_model_backend.cpp: it decodes up to +/// one short of its context, declines the forward after an EOS, aligns +/// prefill chunks, and states the stop ids llama_core_backend.cpp gives it. +class StubCoreBackend final : public flm::backend::ModelBackend { +public: + explicit StubCoreBackend(std::uint32_t context_length) + : context_length_(context_length), + engine_(std::make_unique(context_length)) { + g_factory.engine = engine_.get(); + } + + causal_lm& engine() override { return *engine_; } + std::string id() const override { return flm::backend::kCoreBackendId; } + std::uint32_t max_decode_length() const override { return context_length_ - 1; } + bool supports_preemption() const override { return false; } + bool forwards_past_eos() const override { return false; } + bool aligns_prefill_chunks() const override { return true; } + bool poisoned() const noexcept override { return engine_->poisoned_state; } + std::optional> forced_eos_ids() const override { return kCoreEosIds; } + +private: + std::uint32_t context_length_; + std::unique_ptr engine_; +}; + +std::unique_ptr MakeCoreStub( + const flm::backend::BackendContext& context) { + ++g_factory.calls; + return std::make_unique(context.context_length); +} + +/// \brief a Llama 3 package directory in the shape the catalog pins +/// \param eos_token_id what tokenizer_config.json and config.json list +/// \note No tokenizer.json or GGUF: the tokenizer and the backend are stubs and +/// open neither. +class TempPackage final { +public: + explicit TempPackage(std::vector eos_token_id = kPackageEosIds) { + static std::uint64_t serial{}; + path_ = std::filesystem::temp_directory_path() / + ("flm-llama-frontend-" + std::to_string(++serial)); + std::filesystem::create_directories(path_); + Write(path_ / "config.json", + {{"model_type", "llama"}, {"vocab_size", 128256}, + {"bos_token_id", kBosId}, {"eos_token_id", eos_token_id}}); + Write(path_ / "tokenizer_config.json", + {{"bos_token", "<|begin_of_text|>"}, {"bos_token_id", kBosId}, + {"eos_token", "<|eot_id|>"}, {"eos_token_id", eos_token_id}, + {"chat_template", + "{{ bos_token }}{% for message in messages %}{{ message.content }}{% endfor %}"}}); + } + ~TempPackage() { + std::error_code ignored; + std::filesystem::remove_all(path_, ignored); + } + const std::filesystem::path& path() const { return path_; } + +private: + static void Write(const std::filesystem::path& path, const nlohmann::json& value) { + std::ofstream out(path, std::ios::binary); + if (!out) throw std::runtime_error("cannot write package fixture"); + out << value.dump(); + } + std::filesystem::path path_; +}; + +nlohmann::ordered_json ModelInfo() { + return {{"default_context_length", 4096}, + {"details", {{"family", "llama3"}}}}; +} + +chat_meta_info_t Meta() { + chat_meta_info_t value; + value.max_prefill_len = 64; + return value; +} + +lm_uniform_input_t Input(std::optional budget = std::nullopt) { + lm_uniform_input_t value; + value.prompt = "prompt"; + value.requested_max_new_tokens = budget; + return value; +} + +template +void ExpectRequestError(F&& action, int code, bool cleared, std::string_view text) { + try { action(); } + catch (const ModelRequestError& error) { + TEST_REQUIRE(error.http_code() == code); + TEST_REQUIRE(error.session_cleared() == cleared); + RequireContains(error.what(), text); + return; + } + throw std::runtime_error("expected ModelRequestError"); +} + +} // namespace + +Tokenizer::Tokenizer(const std::string&) { is_doubled_encoded = false; } +Tokenizer::~Tokenizer() = default; +std::vector Tokenizer::encode(const std::string&) { return g_encoded_tokens; } +std::string Tokenizer::decode(const std::vector&) { return "decoded"; } +std::string Tokenizer::run_time_decoder(int token) { return "t" + std::to_string(token); } +SafeTensors::~SafeTensors() = default; + +Sampler::Sampler(int features, sampler_config& config) + : in_features(features), rep_penalty(config.rep_penalty), + freq_penalty(config.freq_penalty), pre_penalty(config.pre_penalty), + top_k(config.top_k), top_p(config.top_p), min_p(config.min_p), + temperature(config.temperature), total_tokens(0), + freq_penalty_window(config.freq_penalty_window), + rep_penalty_window(config.rep_penalty_window), + repeat_last_n(config.repeat_last_n), + use_optimized_sampling(config.use_optimized_sampling) { + logits.resize(1); counters.resize(1); token_positions.resize(1, -1); +} +void Sampler::reset_penalties() {} +int Sampler::sample(buffer&) { + if (g_sample_index < g_samples.size()) return g_samples[g_sample_index++]; + return 7; +} + +namespace { + +/// \brief point the llama3 family's core backend at the stub for one test +/// \note replace_backend, not register_backend: the registry is process-wide +/// and every test re-arms it. +struct FactoryScope { + FactoryScope() { + g_factory = {}; + g_encoded_tokens.clear(); + g_samples.clear(); + g_sample_index = 0; + flm::backend::BackendRegistry::instance().replace_backend( + "llama3", flm::backend::kCoreBackendId, MakeCoreStub, + flm::llama::core_traits()); + } +}; + +std::unique_ptr ReadyCore(const TempPackage& package) { + auto model = std::make_unique(nullptr); + model->load_model(package.path().string(), ModelInfo(), -1, false, + flm::backend::kCoreBackendId); + return model; +} + +void TestRenderedPromptPlusExplicitBudgetMayEqual4095() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens.assign(4000, 1); + auto meta = Meta(); auto input = Input(95); + TEST_REQUIRE(model->insert(meta, input)); +} + +void TestRenderedPromptPlusExplicitBudgetAbove4095Is400() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens.assign(4000, 1); + auto meta = Meta(); auto input = Input(96); + ExpectRequestError([&] { (void)model->insert(meta, input); }, 400, false, "4095"); + TEST_REQUIRE(g_factory.engine->prefill_calls == 0); +} + +void TestAFailedPrefillReturns500AndClearsTheConversation() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens = {1, 2}; g_samples = {11, 12}; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + (void)model->generate(meta, 1, output); + TEST_REQUIRE(!model->get_history().second.empty()); + + g_factory.engine->poison_on_failure = false; + g_factory.engine->fail_prefill = true; + g_encoded_tokens = {1, 2, 11, 3}; + ExpectRequestError([&] { (void)model->insert(meta, input); }, 500, true, + "the current conversation was cleared (submitted inference failed)"); + TEST_REQUIRE(model->get_current_context_length() == 0); + TEST_REQUIRE(model->get_history().second.empty()); + TEST_REQUIRE(g_factory.engine->position == 0); + + // Nothing is left to match against, so the next turn prefills all of it. + g_factory.engine->fail_prefill = false; + g_factory.engine->prefill_sizes.clear(); + auto next = Meta(); + TEST_REQUIRE(model->insert(next, input)); + TEST_REQUIRE((g_factory.engine->prefill_sizes == std::vector{4})); +} + +void TestAFailedDecodeReturns500AndLeavesTheModelPoisoned() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens = {1}; g_samples = {11}; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + g_factory.engine->fail_forward = true; + ExpectRequestError([&] { (void)model->generate(meta, 3, output); }, 500, true, + "unload/reload is required because the model is poisoned " + "(submitted inference failed)"); + TEST_REQUIRE(model->get_current_context_length() == 0); + TEST_REQUIRE(model->get_history().second.empty()); +} + +void TestPoisonedModelReturns500UntilReload() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens = {1}; g_samples = {11}; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + g_factory.engine->fail_forward = true; + ExpectRequestError([&] { (void)model->generate(meta, 3, output); }, 500, true, "unload/reload"); + + // Refused before the engine is touched again. + const int prefills = g_factory.engine->prefill_calls; + const int forwards = g_factory.engine->forward_calls; + ExpectRequestError([&] { (void)model->insert(meta, input); }, 500, true, "unload/reload"); + ExpectRequestError([&] { (void)model->generate(meta, 3, output); }, 500, true, "unload/reload"); + TEST_REQUIRE(g_factory.engine->prefill_calls == prefills); + TEST_REQUIRE(g_factory.engine->forward_calls == forwards); + + auto reloaded = ReadyCore(package); + TEST_REQUIRE(reloaded->insert(meta, input)); +} + +void TestGenerateWithPromptGetsTheSameFailureHandlingAsGenerate() { + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens = {1}; g_samples = {11}; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + g_factory.engine->fail_forward = true; + ExpectRequestError([&] { (void)model->generate_with_prompt(meta, input, 3, output); }, + 500, true, "unload/reload"); + TEST_REQUIRE(model->get_current_context_length() == 0); + TEST_REQUIRE(model->get_history().second.empty()); +} + +void TestEveryStopIdTheBackendStatesEndsTheTurn() { + // The package lists only <|eot_id|>, so <|eom_id|> and <|end_of_text|> can + // only end the turn because the backend states them. + for (const int eos : kCoreEosIds) { + TempPackage package({128009}); FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens = {1, 2}; g_samples = {11, eos}; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + const auto text = model->generate(meta, 10, output); + TEST_REQUIRE(meta.stop_reason == EOT_DETECTED); + TEST_REQUIRE(text == "t11"); + // The core backend declines the forward after an EOS. + TEST_REQUIRE((g_factory.engine->forwarded == std::vector{11})); + } +} + +} // namespace + +int main() { +#define RUN(name) RunTest(&name, #name) + RUN(TestRenderedPromptPlusExplicitBudgetMayEqual4095); + RUN(TestRenderedPromptPlusExplicitBudgetAbove4095Is400); + RUN(TestAFailedPrefillReturns500AndClearsTheConversation); + RUN(TestAFailedDecodeReturns500AndLeavesTheModelPoisoned); + RUN(TestPoisonedModelReturns500UntilReload); + RUN(TestGenerateWithPromptGetsTheSameFailureHandlingAsGenerate); + RUN(TestEveryStopIdTheBackendStatesEndsTheTurn); +#undef RUN + std::cout << "test_llama_frontend: PASS\n"; + return 0; +} diff --git a/src/test/phi4_core/test_phi4_frontend.cpp b/src/test/phi4_core/test_phi4_frontend.cpp index fd63395aa..83eee9611 100644 --- a/src/test/phi4_core/test_phi4_frontend.cpp +++ b/src/test/phi4_core/test_phi4_frontend.cpp @@ -811,6 +811,50 @@ void TestAnEosTheEngineNeverRanIsPrefilledWithTheNextTurn() { } } +void TestATokenCutOffBeforeItRanIsPrefilledWithTheNextTurn() { + // A turn stopped by its budget or a cancel never forwards the token it + // sampled last, so the next turn has to carry it: here 12, 3. The flm + // engines only hold what they were forwarded too, so the same applies. + for (const bool core : {true, false}) { + for (const bool cancel : {false, true}) { + TempPackage package; FactoryScope scope; + auto model = core ? ReadyCore(package) : Load(package, ModelInfo()); + TEST_REQUIRE(OnCore(*model) == core); + g_encoded_tokens = {1, 2}; g_samples = {11, 12}; g_sample_index = 0; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + if (cancel) { + (void)model->generate(meta, 10, output, + [&] { return g_factory.engine->forward_calls >= 1; }); + } else { + (void)model->generate(meta, 1, output); + } + TEST_REQUIRE(meta.stop_reason == (cancel ? CANCEL_DETECTED : MAX_LENGTH_REACHED)); + TEST_REQUIRE((g_factory.engine->forwarded == std::vector{11})); + g_encoded_tokens = {1, 2, 11, 12, 3}; + g_factory.engine->prefill_sizes.clear(); + auto next = Meta(); + TEST_REQUIRE(model->insert(next, input)); + TEST_REQUIRE((g_factory.engine->prefill_sizes == std::vector{2})); + } + } +} + +void TestATokenCutOffByTheDecodeWindowIsLeftOutOfTheHistory() { + // A turn that fills the window has no next turn to prefill, so check the + // history itself: it ends at 12, the last token the engine ran, not at 13. + TempPackage package; FactoryScope scope; auto model = ReadyCore(package); + g_encoded_tokens.assign(4093, 1); g_samples = {11, 12, 13}; g_sample_index = 0; + auto meta = Meta(); auto input = Input(); std::ostringstream output; + TEST_REQUIRE(model->insert(meta, input)); + (void)model->generate(meta, 4096, output); + TEST_REQUIRE(meta.stop_reason == MAX_LENGTH_REACHED); + TEST_REQUIRE((g_factory.engine->forwarded == std::vector{11, 12})); + const auto history = model->get_history().second; + TEST_REQUIRE(history.size() == 4095); + TEST_REQUIRE(history.back() == 12); +} + void TestCoreDecodeTimeAndSpeedAreMeasured() { // The core path used to have a decode loop of its own that recorded nothing, so // profile reported "0 us" and a nan speed. It now shares _shared_generate; @@ -971,6 +1015,8 @@ int main() { RunTest(TestPoisonedModelReturns500UntilReload, "TestPoisonedModelReturns500UntilReload"); RunTest(TestEosSelfTerminatesWithoutAnExtraDecode, "TestEosSelfTerminatesWithoutAnExtraDecode"); RunTest(TestAnEosTheEngineNeverRanIsPrefilledWithTheNextTurn, "TestAnEosTheEngineNeverRanIsPrefilledWithTheNextTurn"); + RunTest(TestATokenCutOffBeforeItRanIsPrefilledWithTheNextTurn, "TestATokenCutOffBeforeItRanIsPrefilledWithTheNextTurn"); + RunTest(TestATokenCutOffByTheDecodeWindowIsLeftOutOfTheHistory, "TestATokenCutOffByTheDecodeWindowIsLeftOutOfTheHistory"); RunTest(TestCoreDecodeTimeAndSpeedAreMeasured, "TestCoreDecodeTimeAndSpeedAreMeasured"); RunTest(TestCliAndAllFourGenerationEndpointsPassTheSameBudgetSemantics, "TestCliAndAllFourGenerationEndpointsPassTheSameBudgetSemantics"); RunTest(TestQueueCompletionIsExactlyOnceAndIncludesCompletionsEndpoint, "TestQueueCompletionIsExactlyOnceAndIncludesCompletionsEndpoint"); From b2b142820a978315482f1834b6db168d2255c719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 26 Oct 2026 23:29:27 -0700 Subject: [PATCH 02/13] test(core): build the corelib frontend harnesses against upstream's chat library and prefix cache automodel.cpp now calls PrefixSlotCache and the chat library, and Sampler::sample takes an allowed-token mask and a logprobs out-parameter. The llama, phi4 and gemma4 frontend harnesses compile prefix_slot_cache.cpp; the llama harness also links flm_xgrammar and stubs the new sample signature. Co-authored-by: Cursor --- src/test/gemma4_core/CMakeLists.txt | 1 + src/test/llama_core/CMakeLists.txt | 10 ++++++++-- src/test/llama_core/test_llama_frontend.cpp | 2 +- src/test/phi4_core/CMakeLists.txt | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/test/gemma4_core/CMakeLists.txt b/src/test/gemma4_core/CMakeLists.txt index f5d8db0d3..f9f9bf8f8 100644 --- a/src/test/gemma4_core/CMakeLists.txt +++ b/src/test/gemma4_core/CMakeLists.txt @@ -60,6 +60,7 @@ if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/test_gemma4_frontend.cpp") "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/modeling_gemma4e.cpp" + "${FLM_SOURCE_DIR}/common/prefix_slot_cache.cpp" ${FLM_CHAT_SOURCES}) target_include_directories(test_gemma4_frontend PRIVATE "${CMAKE_CURRENT_LIST_DIR}" diff --git a/src/test/llama_core/CMakeLists.txt b/src/test/llama_core/CMakeLists.txt index 86152e672..1c1f38b75 100644 --- a/src/test/llama_core/CMakeLists.txt +++ b/src/test/llama_core/CMakeLists.txt @@ -22,15 +22,21 @@ set(FLM_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../..") # builtin_backends.cpp is left out because it names every prebuilt engine # library; test_llama_frontend.cpp supplies its own empty # register_builtin_backends and registers a stub instead. +# automodel.cpp parses output and constrains decoding with the chat library, +# so the target also links flm_xgrammar. +include("${FLM_SOURCE_DIR}/common/chat/chat_sources.cmake") add_executable(test_llama_frontend test_llama_frontend.cpp "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" - "${FLM_SOURCE_DIR}/common/AutoModel/modeling_llama3.cpp") + "${FLM_SOURCE_DIR}/common/AutoModel/modeling_llama3.cpp" + "${FLM_SOURCE_DIR}/common/prefix_slot_cache.cpp" + ${FLM_CHAT_SOURCES}) target_include_directories(test_llama_frontend PRIVATE "${CMAKE_CURRENT_LIST_DIR}" "${FLM_SOURCE_DIR}/include" "${FLM_SOURCE_DIR}/test/phi4_core" + "${FLM_SOURCE_DIR}/../third_party/tokenizers-cpp/include" "${BOOST_INCLUDE_DIR}" "${XRT_INCLUDE_DIR}") target_compile_definitions(test_llama_frontend PRIVATE @@ -39,7 +45,7 @@ target_compile_definitions(test_llama_frontend PRIVATE target_compile_options(test_llama_frontend PRIVATE $<$:/wd4005 /wd4244>) target_link_directories(test_llama_frontend PRIVATE "${XRT_INCLUDE_DIR}/../lib") -target_link_libraries(test_llama_frontend PRIVATE xrt_coreutil) +target_link_libraries(test_llama_frontend PRIVATE xrt_coreutil flm_xgrammar) include(CTest) add_test(NAME test_llama_frontend COMMAND test_llama_frontend) diff --git a/src/test/llama_core/test_llama_frontend.cpp b/src/test/llama_core/test_llama_frontend.cpp index d7ddb580e..05eaa9d8e 100644 --- a/src/test/llama_core/test_llama_frontend.cpp +++ b/src/test/llama_core/test_llama_frontend.cpp @@ -216,7 +216,7 @@ Sampler::Sampler(int features, sampler_config& config) logits.resize(1); counters.resize(1); token_positions.resize(1, -1); } void Sampler::reset_penalties() {} -int Sampler::sample(buffer&) { +int Sampler::sample(buffer&, const std::vector*, sample_logprobs_t*) { if (g_sample_index < g_samples.size()) return g_samples[g_sample_index++]; return 7; } diff --git a/src/test/phi4_core/CMakeLists.txt b/src/test/phi4_core/CMakeLists.txt index 19cff7c0c..68849fcf7 100644 --- a/src/test/phi4_core/CMakeLists.txt +++ b/src/test/phi4_core/CMakeLists.txt @@ -99,6 +99,7 @@ set(PHI4_FRONTEND_SOURCES "${FLM_SOURCE_DIR}/common/AutoModel/automodel.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/model_backend.cpp" "${FLM_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp" + "${FLM_SOURCE_DIR}/common/prefix_slot_cache.cpp" ${FLM_CHAT_SOURCES}) add_executable(test_phi4_frontend From 74c6e4bddff00568eaa880ef8dcc0eccff65a69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 15:46:49 -0700 Subject: [PATCH 03/13] test(core): stub the sampler seed/window setters and the engine state tables in the frontend harnesses modeling_llama3.cpp and modeling_phi4.cpp now attach the prefix cache through llama_npu_state_api / phi4_npu_state_api, which only the engine libraries export; the harnesses return a null table, which leaves the cache off. The llama harness also gets the Sampler::set_seed and set_repeat_last_n stubs the phi4 and gemma4 ones already have. Co-authored-by: Cursor --- src/test/llama_core/test_llama_frontend.cpp | 5 +++++ src/test/phi4_core/test_phi4_frontend.cpp | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/test/llama_core/test_llama_frontend.cpp b/src/test/llama_core/test_llama_frontend.cpp index 05eaa9d8e..28c6ce54c 100644 --- a/src/test/llama_core/test_llama_frontend.cpp +++ b/src/test/llama_core/test_llama_frontend.cpp @@ -216,11 +216,16 @@ Sampler::Sampler(int features, sampler_config& config) logits.resize(1); counters.resize(1); token_positions.resize(1, -1); } void Sampler::reset_penalties() {} +void Sampler::set_seed(uint64_t) {} +void Sampler::set_repeat_last_n(size_t n) { repeat_last_n = n; } int Sampler::sample(buffer&, const std::vector*, sample_logprobs_t*) { if (g_sample_index < g_samples.size()) return g_samples[g_sample_index++]; return 7; } +// No engine library is linked; a null table leaves the prefix cache off. +extern "C" const flm_state_api* llama_npu_state_api(void*) { return nullptr; } + namespace { /// \brief point the llama3 family's core backend at the stub for one test diff --git a/src/test/phi4_core/test_phi4_frontend.cpp b/src/test/phi4_core/test_phi4_frontend.cpp index 83eee9611..605a477a6 100644 --- a/src/test/phi4_core/test_phi4_frontend.cpp +++ b/src/test/phi4_core/test_phi4_frontend.cpp @@ -314,6 +314,9 @@ int Sampler::sample(buffer& logits, const std::vector*, sample_l return 7; } +// No engine library is linked; a null table leaves the prefix cache off. +extern "C" const flm_state_api* phi4_npu_state_api(void*) { return nullptr; } + namespace utils { std::string get_executable_directory() { return std::filesystem::current_path().string(); } } From d9b0ae43efb4c8e0bf648bf5060c36b4352dd60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 17:02:53 -0700 Subject: [PATCH 04/13] feat(core): move to ryzenai-corelib 0.53.0 corelib splits a model from its sessions (0.48.0): create_model_from_path replaces model_weights_create_path, model_create_session replaces model_create, and forward, reset, get_info, state_save and state_restore take the session. CorelibModel keeps one model for the engine's life and opens a new session of it when the capacity grows, so a larger context no longer re-reads the weights. model_session_options is model_create_options renamed, with lora and enabled_backends appended; zero leaves both at their defaults. model_info_get is model_get_info (0.53.0). flat_mha_bf16, its enum_kernels and paged_mha_bf16_enum_kernels become one mha_bf16 (0.37.0). Nothing here called them, but the function table resolves every symbol at load, so they leave it. ryzenai_corelib_get_hardware_context goes (0.53.0): each stream opens its own hardware context on a device corelib keeps to itself. The process no longer takes corelib's xrt::device; it opens its own for every other family, and a core model shares no buffer with those engines. corelib_device.hpp goes, and main.cpp keeps its XRT-only #error. The runtime is ready only where ryzenai_corelib_npu_usable says so: on a refused NPU stack corelib now runs a session on the GPU or the host instead of failing. The fake corelib follows every signature and tells a model from a session, both an object_ptr to the compiler, refusing one passed for the other as corelib would. The docs name 0.53.0; core-setup-linux.md adds RYZENAI_CORELIB_DISABLE_GPU and RYZENAI_CORELIB_PROFILE and drops RYZENAI_CORELIB_NO_RUN_POOL and RYZENAI_CORELIB_MAX_RUNS_IN_FLIGHT, which corelib no longer reads. Co-authored-by: Cursor --- docs/core-setup-linux.md | 10 +- docs/docs/models/phi.md | 4 +- docs/docs/models/qwen.md | 2 +- src/common/core/corelib_model.cpp | 41 ++++---- src/common/core/corelib_runtime.cpp | 8 +- src/common/models/README.md | 2 +- src/include/core/corelib_api.hpp | 17 ++-- src/include/core/corelib_device.hpp | 41 -------- src/include/core/corelib_object.hpp | 4 +- .../qwen36/core/dev04/qwen36_core_backend.hpp | 4 +- src/src/main.cpp | 62 ++++-------- src/test/core_model/test_corelib_model.cpp | 28 +++--- .../core_model/test_corelib_model_backend.cpp | 6 +- .../core_model/test_llama_core_backend.cpp | 4 +- .../core_model/test_phi4_core_backend.cpp | 4 +- src/test/phi4_core/fake_corelib.cpp | 97 +++++++------------ src/test/phi4_core/fake_corelib.hpp | 19 +--- src/test/phi4_core/test_corelib_api.cpp | 8 +- 18 files changed, 127 insertions(+), 234 deletions(-) delete mode 100644 src/include/core/corelib_device.hpp diff --git a/docs/core-setup-linux.md b/docs/core-setup-linux.md index 2d2ff7677..065fb8efc 100644 --- a/docs/core-setup-linux.md +++ b/docs/core-setup-linux.md @@ -9,7 +9,7 @@ Written against the working install on this machine. Paths use `$P` for the dependency prefix (here `/home/alfxu/ddprefix`); substitute your own. > **Status:** the core backend has been run on hardware on Windows only. The -> steps below are updated for corelib 0.33.1, but they have not been run end +> steps below are updated for corelib 0.53.0, but they have not been run end > to end on Linux at this revision. The [Known issues](#known-issues) are from > an earlier Linux bring-up. @@ -52,7 +52,7 @@ loaded from, so those libraries must sit beside it; nothing else is needed at run time, no kernel store and no environment variable. **Version must match exactly.** `src/include/core/corelib_api.hpp` has a -`#error` for anything other than corelib **0.33.1**: +`#error` for anything other than corelib **0.53.0**: ```bash grep -E 'CORELIB_VERSION_(MAJOR|MINOR|PATCH)' \ @@ -235,10 +235,10 @@ shape is a dead end. |---|---| | `RYZENAI_CORELIB_TRACE=xrt` | trace XRT calls (`kernels,xrt` for both logs, `all` for every category) | | `RYZENAI_CORELIB_SAFE_MODE` | force staging buffers, no host-pointer import | -| `RYZENAI_CORELIB_NO_RUN_POOL` | disable run-object reuse | | `RYZENAI_CORELIB_NO_CRASH_HANDLER` | leave crashes to the debugger | -| `RYZENAI_CORELIB_REPORT_FALLBACKS=1` | when a stream is released, report host (CPU fallback) time per artifact, and every dispatch that ran on a later PDI than its first choice; no output means nothing fell back | -| `RYZENAI_CORELIB_MAX_RUNS_IN_FLIGHT` | how many runs of one kernel a stream lets run ahead (64 unset, `0` for no cap) | +| `RYZENAI_CORELIB_REPORT_FALLBACKS=1` | when a stream is released, report host and GPU time per artifact, and every dispatch that ran on a later PDI than its first choice; no output means nothing fell back | +| `RYZENAI_CORELIB_DISABLE_GPU=1` | keep streams created while it is set off the APU's GPU, so whatever no NPU kernel takes runs on the host | +| `RYZENAI_CORELIB_PROFILE` | `1` prints a per-phase device-time report when a stream is released; a path also writes it as JSON | | `RYZENAI_CORELIB_KERNELS_ROOT` | a kernel store to load from, for corelib development only; every model library already carries its kernels, so a normal run sets nothing | ### Other rough edges diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index defee709d..60e37026c 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -75,9 +75,9 @@ The `src/test/phi4_core` suite is still Windows-only and is not configured on Li ### Pointing FastFlowLM at the runtime -A core build (`-DFLM_ENABLE_CORE=ON`) **links corelib in**, because the NPU device the whole process shares is the one behind corelib's hardware context rather than a device FastFlowLM opens itself. Point the build at the library with `RYZENAI_CORELIB_INCLUDE_DIR` and `RYZENAI_CORELIB_LIBRARY`. `FLM_CORE_CORELIB_PATH` selects a shared library (`.dll` on Windows, `.so` elsewhere) only in the older dynamically loading configuration; in a statically linked core build it is ignored, and `flm` says so if it is set. +A core build (`-DFLM_ENABLE_CORE=ON`) **links corelib in**. corelib keeps its NPU device to itself and opens a hardware context per stream on it; FastFlowLM opens its own device for every other family, and a core model shares no buffer with those. The core backend is offered only where corelib reports a usable NPU (`ryzenai_corelib_npu_usable`): elsewhere corelib would run a session on the GPU or the host. Point the build at the library with `RYZENAI_CORELIB_INCLUDE_DIR` and `RYZENAI_CORELIB_LIBRARY`. `FLM_CORE_CORELIB_PATH` selects a shared library (`.dll` on Windows, `.so` elsewhere) only in the older dynamically loading configuration; in a statically linked core build it is ignored, and `flm` says so if it is set. -The corelib ABI is still pre-1.0, so FastFlowLM requires an **exact `0.33.1`** match on major, minor and patch. The version is queried before any other entry point, so a mismatched runtime reports a version error rather than a missing symbol. corelib's default build installs `ryzenai_corelib` plus one `ryzenai_corelib_model_` library per model, each carrying that model's kernels; corelib loads them only from its own directory, so they must sit beside it (a Windows core build copies them next to `flm.exe`). Each engine enables its model's kernel bundle on the stream it opens; no kernel store or environment variable is involved. Corelib's own dependency directory must be reachable on `PATH` (Windows) or `LD_LIBRARY_PATH` (Linux). +The corelib ABI is still pre-1.0, so FastFlowLM requires an **exact `0.53.0`** match on major, minor and patch. The version is queried before any other entry point, so a mismatched runtime reports a version error rather than a missing symbol. corelib's default build installs `ryzenai_corelib` plus one `ryzenai_corelib_model_` library per model, each carrying that model's kernels; corelib loads them only from its own directory, so they must sit beside it (a Windows core build copies them next to `flm.exe`). Each engine enables its model's kernel bundle on the stream it opens; no kernel store or environment variable is involved. Corelib's own dependency directory must be reachable on `PATH` (Windows) or `LD_LIBRARY_PATH` (Linux). ```powershell flm pull phi4-mini-it-core:4b diff --git a/docs/docs/models/qwen.md b/docs/docs/models/qwen.md index 2c1b097ee..f58b3555a 100644 --- a/docs/docs/models/qwen.md +++ b/docs/docs/models/qwen.md @@ -107,7 +107,7 @@ flm run qwen3:8b - **Backend:** `core` — one engine for all four sizes. The GGUF's own metadata selects the size and is cross-checked against it. - **Source format:** GGUF `Q8_0`, read directly. Every weight is requantized to **group 64** while it is packed, except the 0.6B lm_head, which packs at **group 32**. - **Usable generation window:** 4095 tokens, prompt plus output. Over-capacity requests are rejected with HTTP 400 before any work reaches the device. -- **Availability:** developer build against corelib 0.33.1 — see [Phi-4 on the core backend](phi.md). +- **Availability:** developer build against corelib 0.53.0 — see [Phi-4 on the core backend](phi.md). The same corelib path also runs `gemma4-it-core:e2b` and `gemma4-it-core:e4b` (text only), `qwen3.5-core:9b` and `qwen3.6-moe-core:35b-a3b` (text only). Qwen3 stays at 4096; the other three go further. diff --git a/src/common/core/corelib_model.cpp b/src/common/core/corelib_model.cpp index a5b43ff81..7021ab09b 100644 --- a/src/common/core/corelib_model.cpp +++ b/src/common/core/corelib_model.cpp @@ -19,8 +19,8 @@ namespace flm::core { using corelib::CorelibApi; using corelib::CorelibRuntime; using corelib::UniqueModel; +using corelib::UniqueModelSession; using corelib::UniqueModelSnapshot; -using corelib::UniqueModelWeights; struct CorelibModel::Impl { Impl(ModelSpec model_spec, const std::filesystem::path& file, @@ -43,10 +43,10 @@ struct CorelibModel::Impl { auto lease = runtime->AcquireExecution(); const auto path = file.string(); void* raw = nullptr; - api->Check(api->functions().model_weights_create_path(spec.corelib_name.c_str(), - path.c_str(), &raw), - "ryzenai_corelib_model_weights_create_path"); - weights = UniqueModelWeights(api, raw); + api->Check(api->functions().create_model_from_path(spec.corelib_name.c_str(), + path.c_str(), &raw), + "ryzenai_corelib_create_model_from_path"); + model = UniqueModel(api, raw); Recreate(maximum); } @@ -59,8 +59,8 @@ struct CorelibModel::Impl { ModelSpec spec; std::shared_ptr runtime; std::shared_ptr api; - UniqueModelWeights weights; UniqueModel model; + UniqueModelSession session; std::uint32_t capacity = 0; std::uint32_t max_length = 0; std::size_t vocab = 0; @@ -192,7 +192,7 @@ struct CorelibModel::Impl { float* out_logits = want_logits ? logits.data() : nullptr; const std::size_t logits_capacity = want_logits ? logits.size() : 0; try { - api->Check(api->functions().model_forward(model.get(), rows.data(), nullptr, + api->Check(api->functions().model_forward(session.get(), rows.data(), nullptr, rows.size(), nullptr, 0, &options, nullptr, out_logits, logits_capacity), "ryzenai_corelib_model_forward"); @@ -219,8 +219,8 @@ struct CorelibModel::Impl { if (!target.empty()) Advance(target, false); } - /// \note The capacity is fixed for a model's life, so a larger one is a - /// new model over the same packed weights. + /// \note The capacity is fixed for a session's life, so a larger one is a + /// new session of the same model. void Grow(std::uint32_t n) { auto lease = runtime->AcquireExecution(); UniqueModelSnapshot current; @@ -236,24 +236,23 @@ struct CorelibModel::Impl { } void Recreate(std::uint32_t n) { - ryzenai_corelib_model_create_options options{}; + ryzenai_corelib_model_session_options options{}; options.size = sizeof(options); options.max_sequence_length = static_cast(n); void* raw = nullptr; - api->Check(api->functions().model_create(spec.corelib_name.c_str(), weights.get(), - &options, &raw), - "ryzenai_corelib_model_create"); - UniqueModel created(api, raw); + api->Check(api->functions().model_create_session(model.get(), &options, &raw), + "ryzenai_corelib_model_create_session"); + UniqueModelSession created(api, raw); ryzenai_corelib_model_info info{}; - api->Check(api->functions().model_info_get(created.get(), &info), - "ryzenai_corelib_model_info_get"); + api->Check(api->functions().model_get_info(created.get(), &info), + "ryzenai_corelib_model_get_info"); if (info.vocab_size <= 0 || info.context_length < static_cast(n)) throw std::runtime_error(spec.display_name + " model_info is vocab " + std::to_string(info.vocab_size) + " context " + std::to_string(info.context_length) + "; this engine needs vocab > 0 and context >= " + std::to_string(n)); - model = std::move(created); + session = std::move(created); capacity = static_cast(info.context_length); vocab = static_cast(info.vocab_size); logits.resize(vocab); @@ -261,16 +260,16 @@ struct CorelibModel::Impl { UniqueModelSnapshot Save() { void* raw = nullptr; - api->Check(api->functions().model_state_save(model.get(), &raw), + api->Check(api->functions().model_state_save(session.get(), &raw), "ryzenai_corelib_model_state_save"); return UniqueModelSnapshot(api, raw); } - /// \note corelib leaves the model reset when a restore fails partway, which + /// \note corelib leaves the session reset when a restore fails partway, which /// no history here describes. void Load(const UniqueModelSnapshot& snapshot) { try { - api->Check(api->functions().model_state_restore(model.get(), snapshot.get()), + api->Check(api->functions().model_state_restore(session.get(), snapshot.get()), "ryzenai_corelib_model_state_restore"); } catch (...) { Poison(); @@ -280,7 +279,7 @@ struct CorelibModel::Impl { void ResetModel() { try { - api->Check(api->functions().model_reset(model.get()), "ryzenai_corelib_model_reset"); + api->Check(api->functions().model_reset(session.get()), "ryzenai_corelib_model_reset"); } catch (...) { Poison(); throw; diff --git a/src/common/core/corelib_runtime.cpp b/src/common/core/corelib_runtime.cpp index 8eae04aba..30fd8f103 100644 --- a/src/common/core/corelib_runtime.cpp +++ b/src/common/core/corelib_runtime.cpp @@ -27,10 +27,10 @@ std::shared_ptr CorelibRuntime::CreateReady( if (!api) throw std::invalid_argument("corelib API is null"); api->Check(api->functions().selftest_dependencies(), "ryzenai_corelib_selftest_dependencies"); - // NULL means this process has no NPU hardware context to dispatch on. - // The bound symbol is ryzenai_corelib_get_hardware_context. - if (api->functions().get_device() == nullptr) { - throw std::runtime_error("corelib has no device context"); + // Without one corelib still runs a session, on the GPU or the host, which + // is the silent slowness the core backend is meant to rule out. + if (!api->functions().npu_usable()) { + throw std::runtime_error("corelib has no usable NPU"); } return std::shared_ptr(new CorelibRuntime(std::move(api))); } diff --git a/src/common/models/README.md b/src/common/models/README.md index b5876385f..f382c69e1 100644 --- a/src/common/models/README.md +++ b/src/common/models/README.md @@ -45,7 +45,7 @@ This is a contributor document. For *using* a backend once it exists — `--back |---|---| | **Platform** | Windows and Linux. Both configure and build; only Windows has been run on hardware, and the `src/test/phi4_core` suite is still Windows-only. | | **Hardware** | An dev04 NPU. There is no simulator; a wrong shape shows up as garbage output, not an error. | -| **corelib headers** | The corelib ABI is still pre-1.0, so FastFlowLM requires an **exact `0.33.1`** match on major, minor and patch. [`corelib_api.hpp`](../../include/core/corelib_api.hpp) `#error`s on any other version — deliberately, because the C ABI has changed shape between patch releases. The version is queried before any other entry point, so a mismatched runtime reports a version error rather than a missing symbol. corelib's default build gives one `ryzenai_corelib_model_` library per model, carrying its kernels, which must sit beside corelib. A per-op engine enables its model's kernel bundle on the stream it opens; a model corelib runs whole enables its own. | +| **corelib headers** | The corelib ABI is still pre-1.0, so FastFlowLM requires an **exact `0.53.0`** match on major, minor and patch. [`corelib_api.hpp`](../../include/core/corelib_api.hpp) `#error`s on any other version — deliberately, because the C ABI has changed shape between patch releases. The version is queried before any other entry point, so a mismatched runtime reports a version error rather than a missing symbol. corelib's default build gives one `ryzenai_corelib_model_` library per model, carrying its kernels, which must sit beside corelib. A per-op engine enables its model's kernel bundle on the stream it opens; a model corelib runs whole enables its own. | | **Weights** | A GGUF corelib's model reads: a Q8_0 one, which it requantizes on every load, or a Q4_G128 or Q4_G32 one, whose codes it packs as they are. The catalog names which; see `preferred_source` in §6. | Configure with: diff --git a/src/include/core/corelib_api.hpp b/src/include/core/corelib_api.hpp index ad08c7760..45e23dc06 100644 --- a/src/include/core/corelib_api.hpp +++ b/src/include/core/corelib_api.hpp @@ -2,9 +2,9 @@ #include -#if RYZENAI_CORELIB_VERSION_MAJOR != 0 || RYZENAI_CORELIB_VERSION_MINOR != 33 || \ - RYZENAI_CORELIB_VERSION_PATCH != 1 -#error "FastFlowLM requires ryzenai-corelib headers exactly 0.33.1" +#if RYZENAI_CORELIB_VERSION_MAJOR != 0 || RYZENAI_CORELIB_VERSION_MINOR != 53 || \ + RYZENAI_CORELIB_VERSION_PATCH != 0 +#error "FastFlowLM requires ryzenai-corelib headers exactly 0.53.0" #endif #include @@ -22,7 +22,7 @@ X(status_to_string, ryzenai_corelib_status_to_string) \ X(get_last_error_message, ryzenai_corelib_get_last_error_message) \ X(selftest_dependencies, ryzenai_corelib_selftest_dependencies) \ - X(get_device, ryzenai_corelib_get_hardware_context) \ + X(npu_usable, ryzenai_corelib_npu_usable) \ X(object_release, ryzenai_corelib_object_release) \ X(create_stream, ryzenai_corelib_create_stream) \ X(stream_synchronize, ryzenai_corelib_stream_synchronize) \ @@ -40,14 +40,11 @@ X(ssmlp_weights_create, ryzenai_corelib_ssmlp_bf16_weights_create) \ X(weights_copy_data, ryzenai_corelib_weights_copy_data) \ X(ssmlp, ryzenai_corelib_ssmlp_bf16) \ - X(flat_mha_enum_kernels, ryzenai_corelib_flat_mha_bf16_enum_kernels) \ - X(flat_mha, ryzenai_corelib_flat_mha_bf16) \ - X(paged_mha_enum_kernels, ryzenai_corelib_paged_mha_bf16_enum_kernels) \ - X(model_info_get, ryzenai_corelib_model_info_get) \ + X(model_get_info, ryzenai_corelib_model_get_info) \ X(model_forward, ryzenai_corelib_model_forward) \ X(model_reset, ryzenai_corelib_model_reset) \ - X(model_weights_create_path, ryzenai_corelib_model_weights_create_path) \ - X(model_create, ryzenai_corelib_model_create) \ + X(create_model_from_path, ryzenai_corelib_create_model_from_path) \ + X(model_create_session, ryzenai_corelib_model_create_session) \ X(model_state_save, ryzenai_corelib_model_state_save) \ X(model_state_restore, ryzenai_corelib_model_state_restore) \ X(cleanup, ryzenai_corelib_cleanup) diff --git a/src/include/core/corelib_device.hpp b/src/include/core/corelib_device.hpp deleted file mode 100644 index c29bd19ef..000000000 --- a/src/include/core/corelib_device.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/// \file corelib_device.hpp -/// \brief Access to the NPU device that ryzenai-corelib owns. -/// \note The C ABI exposes the hardware context -/// (ryzenai_corelib_get_hardware_context), which is corelib's -/// xrt::hw_context. The device is the shared handle that context was -/// opened on. A null hardware context means there is no NPU; casting -/// that pointer to an xrt::device would hand the engines the wrong -/// object. -#pragma once - -#if defined(FLM_USE_HRX) -#error "FLM_ENABLE_CORE requires the XRT backend (FLM_USE_HRX=OFF)" -#endif - -#include "device_runtime.hpp" -#include "core/corelib_runtime.hpp" - -#include "xrt/xrt_hw_context.h" - -namespace flm::corelib { - -/// \brief the device corelib dispatches on, shared with every engine here -/// \param runtime an initialized corelib runtime -/// \return corelib's device, or nullptr when this machine has no NPU -/// \note The returned pointer stays valid for the process. hw_context::get_device -/// returns the handle by value, so a copy is kept here; it is the same -/// NPU corelib dispatches on, not a second open. -/// \note flm_rt is an alias for xrt on this path. The engines want the device -/// mutable. A buffer object created against a different xrt::device for -/// the same NPU binds without error and then never completes. -inline flm_rt::device* SharedDevice(const CorelibRuntime& runtime) { - const void* hardware = runtime.api()->functions().get_device(); - if (hardware == nullptr) { - return nullptr; - } - static flm_rt::device device = - static_cast(hardware)->get_device(); - return &device; -} - -} // namespace flm::corelib diff --git a/src/include/core/corelib_object.hpp b/src/include/core/corelib_object.hpp index 6805d6920..6a81b61cf 100644 --- a/src/include/core/corelib_object.hpp +++ b/src/include/core/corelib_object.hpp @@ -15,7 +15,7 @@ struct HostViewTag {}; struct MatMulWeightsTag {}; struct SsMlpWeightsTag {}; struct ModelTag {}; -struct ModelWeightsTag {}; +struct ModelSessionTag {}; struct ModelSnapshotTag {}; template @@ -67,7 +67,7 @@ using UniqueHostView = UniqueObject; using UniqueMatMulWeights = UniqueObject; using UniqueSsMlpWeights = UniqueObject; using UniqueModel = UniqueObject; -using UniqueModelWeights = UniqueObject; +using UniqueModelSession = UniqueObject; using UniqueModelSnapshot = UniqueObject; /// \brief a stream on one PDI per phase that resolves the kernels `bundle` ships diff --git a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp index 719ce2754..3e9c0bb1d 100644 --- a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp +++ b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp @@ -9,8 +9,8 @@ namespace flm::qwen36 { -/// \note corelib refuses a capacity past 131072 at model_create, below the -/// model's own 262144: the most it has verified on the NPU. +/// \note corelib takes up to the model's own 262144; this stays at 131072, +/// the most FastFlowLM has run on the NPU. inline constexpr std::uint32_t kCoreContextLimit = 131072; inline flm::backend::BackendTraits core_traits() { diff --git a/src/src/main.cpp b/src/src/main.cpp index 814fb524c..7ae61a332 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -38,8 +38,10 @@ #include #include "benchmarking.hpp" #ifdef FLM_ENABLE_CORE +#if defined(FLM_USE_HRX) +#error "FLM_ENABLE_CORE requires the XRT backend (FLM_USE_HRX=OFF): corelib opens the NPU through XRT" +#endif #include "core/corelib_runtime.hpp" -#include "core/corelib_device.hpp" #endif #ifndef _WIN32 @@ -598,8 +600,6 @@ int main(int argc, char* argv[]) { // One NPU runtime for the whole process, brought up before the catalog so // the catalog can be filtered to what this NPU generation can actually run. - // Which runtime that is follows from the build, not from a decision here: - // the flm backend opens its own device, core gets one from corelib. #ifdef FLM_ENABLE_CORE // Declared before anything that can return so the destructor covers the // early exits and the catch-all below. @@ -612,51 +612,25 @@ int main(int argc, char* argv[]) { // is left with "NPU device instance is nullptr" -- raised much later, by // whichever model reaches for the device first -- and nothing to act on. std::string npu_open_error; - // Set when corelib failed but a direct device was opened anyway: the core - // backend is gone, the rest of the build is not. + // Set when corelib failed: the core backend is gone, the rest of the build + // is not. std::string corelib_note; try { - const auto runtime = - flm::corelib::CorelibRuntime::GetOrCreate(std::filesystem::path(exe_dir)); - // corelib opens the NPU for this process, so take its device rather - // than opening a second one: a buffer object created against a - // different xrt::device for the same NPU binds without error and then - // never completes. GetOrCreate holds the runtime process-wide, so the - // device stays valid until CoreProcessGuard tears it down at exit. - npu_device = flm::corelib::SharedDevice(*runtime); - if (npu_device == nullptr) - npu_open_error = - "corelib started but reports no NPU device on this machine"; + // corelib serves the core models alone and keeps its device to itself, + // opening a hardware context per stream on it, so the engines here get + // the device opened below. A core model shares no buffer with them. + (void)flm::corelib::CorelibRuntime::GetOrCreate(std::filesystem::path(exe_dir)); } catch (const std::exception& e) { - // A box with no NPU must still run `flm list`/`pull`/`version`, so this - // stays a null device rather than an error, as in the non-core path. - npu_open_error = std::string("corelib unavailable: ") + e.what(); - header_print("FLM", "corelib unavailable: " << e.what()); + corelib_note = std::string("corelib unavailable: ") + e.what(); } + // A box with no NPU must still run `flm list`/`pull`/`version`, so this + // stays a null device rather than an error, as in the non-core path. + std::string raw_error; + npu_device = open_npu_device(&raw_error); if (npu_device == nullptr) { - // corelib having no device is a reason for the core backend to be - // unavailable, not for the process to have no NPU at all. Every other - // family is served by the flm backend, which opens its own device and - // never touches corelib -- so opening one directly keeps those models - // working instead of failing the whole process over a backend they do - // not use. On a box whose NPU has no creatable DEV04 hw_context this is - // the difference between "core is unavailable" and "flm has no NPU". - // - // This does not bring back the two-device bug, where a buffer created - // against a second xrt::device for the same NPU bound without error and - // then never completed. That needed corelib to be live and holding a - // device of its own; here it has none, so the process still ends up - // with exactly one. - corelib_note = npu_open_error; - std::string raw_error; - npu_device = open_npu_device(&raw_error); - if (npu_device != nullptr) { - npu_open_error.clear(); - } else { - npu_open_error += "; opening the NPU directly also failed: " + raw_error; - corelib_note.clear(); - } + npu_open_error = corelib_note.empty() ? raw_error : corelib_note + "; " + raw_error; + corelib_note.clear(); } #else // validate probes /dev/accel/* itself, and amdxdna refuses a second open of @@ -693,9 +667,7 @@ int main(int argc, char* argv[]) { // rather than the cause: it is raised from whichever model first reaches // for the device, long after the runtime that failed to come up. if (print_status && needs_npu && !corelib_note.empty()) { - header_print("FLM", "core backend unavailable (" - << corelib_note - << "); opened the NPU directly"); + header_print("FLM", "core backend unavailable (" << corelib_note << ")"); } // Only print early error for RAI builds where the device is opened eagerly. #ifdef FLM_ENABLE_RAI diff --git a/src/test/core_model/test_corelib_model.cpp b/src/test/core_model/test_corelib_model.cpp index 1aa5e39a9..f1c23355c 100644 --- a/src/test/core_model/test_corelib_model.cpp +++ b/src/test/core_model/test_corelib_model.cpp @@ -62,10 +62,11 @@ int Resets() { return S().call_counts["ryzenai_corelib_model_reset"]; } // --- construction ----------------------------------------------------------- -void TestTheWeightsAndTheModelAreCorelibsOwnByName() { +void TestTheModelIsCorelibsOwnByNameWithOneSession() { Harness h(kFromStart, 4096); - TEST_REQUIRE((S().model_names == std::vector{"qwen3", "qwen3"})); - TEST_REQUIRE(S().model_weights_path == "C:/fake/model.gguf"); + TEST_REQUIRE((S().model_names == std::vector{"qwen3"})); + TEST_REQUIRE(S().model_path == "C:/fake/model.gguf"); + TEST_REQUIRE(S().session_creates == 1); } void TestTheModelIsCreatedAtTheConfiguredCapacity() { @@ -249,21 +250,21 @@ void TestShrinkingBelowTheCheckpointDropsIt() { TEST_REQUIRE(S().state_restores == 0); } -void TestGrowingASnapshotModelSavesAndRestoresAcrossTheNewModel() { +void TestGrowingASnapshotModelSavesAndRestoresAcrossTheNewSession() { Harness h(kSnapshot, 4096); auto ids = Ids(10); (void)h.engine->prefill(ids); h.engine->update_max_length(16384); - TEST_REQUIRE(S().model_creates == 2); + TEST_REQUIRE(S().session_creates == 2 && S().model_names.size() == 1); TEST_REQUIRE(S().model_max_sequence_length == 16384); TEST_REQUIRE(S().state_saves == 1 && S().state_restores == 1); TEST_REQUIRE(h.engine->get_current_context_length() == 10); } -void TestShrinkingWithinTheCapacityKeepsTheModel() { +void TestShrinkingWithinTheCapacityKeepsTheSession() { Harness h(kSnapshot, 16384); h.engine->update_max_length(8192); - TEST_REQUIRE(S().model_creates == 1); + TEST_REQUIRE(S().session_creates == 1); } // --- Rewind::kSeek ------------------------------------------------------------- @@ -363,12 +364,13 @@ void TestAPagedContinuationOnABoundaryRunsOnlyTheNewRows() { TEST_REQUIRE(Last().position == 4096 && Last().rows == 2); } -void TestGrowingASeekModelReplaysIntoTheNewModel() { +void TestGrowingASeekModelReplaysIntoTheNewSession() { Harness h(kFromPage, 4096); auto ids = Ids(10); (void)h.engine->prefill(ids); h.engine->update_max_length(16384); - TEST_REQUIRE(S().model_creates == 2 && S().state_saves == 0); + TEST_REQUIRE(S().session_creates == 2 && S().model_names.size() == 1 && + S().state_saves == 0); TEST_REQUIRE(Last().rows == 10 && Last().position == 0 && !Last().logits); TEST_REQUIRE(h.engine->get_current_context_length() == 10); } @@ -440,7 +442,7 @@ void TestClearingAReplayModelResets() { int main() { #define RUN(name) RunTest(&name, #name) - RUN(TestTheWeightsAndTheModelAreCorelibsOwnByName); + RUN(TestTheModelIsCorelibsOwnByNameWithOneSession); RUN(TestTheModelIsCreatedAtTheConfiguredCapacity); RUN(TestTheSpecsLimitIsAccepted); RUN(TestAMaximumOutsideTheSpecIsRefused); @@ -459,8 +461,8 @@ int main() { RUN(TestCheckpointingTheSameStateTwiceSavesOnce); RUN(TestRestoringRightAfterACheckpointLoadsNothing); RUN(TestShrinkingBelowTheCheckpointDropsIt); - RUN(TestGrowingASnapshotModelSavesAndRestoresAcrossTheNewModel); - RUN(TestShrinkingWithinTheCapacityKeepsTheModel); + RUN(TestGrowingASnapshotModelSavesAndRestoresAcrossTheNewSession); + RUN(TestShrinkingWithinTheCapacityKeepsTheSession); RUN(TestASeekModelStatesWhereEveryPassStarts); RUN(TestAFromStartContinuationRerunsTheWholeSequence); RUN(TestASeekRewindMovesOnlyThePosition); @@ -470,7 +472,7 @@ int main() { RUN(TestAnUnpagedFromPageContinuationRerunsFromZero); RUN(TestAPagedContinuationRerunsOnlyItsPage); RUN(TestAPagedContinuationOnABoundaryRunsOnlyTheNewRows); - RUN(TestGrowingASeekModelReplaysIntoTheNewModel); + RUN(TestGrowingASeekModelReplaysIntoTheNewSession); RUN(TestAReplayCheckpointKeepsNoSnapshot); RUN(TestAReplayRestoreResetsAndReplays); RUN(TestAReplayRewindResetsAndReplays); diff --git a/src/test/core_model/test_corelib_model_backend.cpp b/src/test/core_model/test_corelib_model_backend.cpp index 06cc74fa1..9653625d6 100644 --- a/src/test/core_model/test_corelib_model_backend.cpp +++ b/src/test/core_model/test_corelib_model_backend.cpp @@ -82,7 +82,7 @@ BackendContext Context(const Package& package, std::uint32_t length, } int WeightLoads() { - return fake_corelib::GetState().call_counts["ryzenai_corelib_model_weights_create_path"]; + return fake_corelib::GetState().call_counts["ryzenai_corelib_create_model_from_path"]; } void TestPreemptionIsRefusedBeforeTheWeightsAreOpened() { @@ -123,7 +123,7 @@ void TestTheCatalogsGgufIsTheOneLoaded() { Package p({"a.gguf", "b.gguf"}); auto backend = flm::core::Factory(Spec())( Context(p, 4096, nlohmann::ordered_json::array({"b.gguf", "config.json"}))); - TEST_REQUIRE(fake_corelib::GetState().model_weights_path == (p.root / "b.gguf").string()); + TEST_REQUIRE(fake_corelib::GetState().model_path == (p.root / "b.gguf").string()); } void TestACatalogGgufThatIsMissingIsNamed() { @@ -152,7 +152,7 @@ void TestWithoutACatalogGgufTheDirectorysOnlyOneIsLoaded() { Package p({"only.gguf", "config.json"}); auto backend = flm::core::Factory(Spec())( Context(p, 4096, nlohmann::ordered_json::array({"config.json"}))); - TEST_REQUIRE(fake_corelib::GetState().model_weights_path == (p.root / "only.gguf").string()); + TEST_REQUIRE(fake_corelib::GetState().model_path == (p.root / "only.gguf").string()); } void TestWithoutACatalogGgufTwoInTheDirectoryAreRefused() { diff --git a/src/test/core_model/test_llama_core_backend.cpp b/src/test/core_model/test_llama_core_backend.cpp index e5c718cd3..f00ebd524 100644 --- a/src/test/core_model/test_llama_core_backend.cpp +++ b/src/test/core_model/test_llama_core_backend.cpp @@ -73,7 +73,7 @@ void TestTheCatalogsGgufRunsAsCorelibsLlama() { auto backend = flm::llama::core_factory()(Context(p, flm::llama::kCoreContextLimit)); const auto& state = fake_corelib::GetState(); TEST_REQUIRE(state.model_names.front() == "llama"); - TEST_REQUIRE(state.model_weights_path == (p.root / kGguf).string()); + TEST_REQUIRE(state.model_path == (p.root / kGguf).string()); TEST_REQUIRE(state.model_max_sequence_length == 4096); TEST_REQUIRE(backend->id() == flm::backend::kCoreBackendId); TEST_REQUIRE(backend->max_decode_length() == 4095); @@ -90,7 +90,7 @@ void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { RequireContains(RequireThrows( [&] { (void)flm::llama::core_factory()(Context(p, 4097)); }), "1..4096"); - TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_model_weights_create_path"] == 0); + TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_create_model_from_path"] == 0); } } // namespace diff --git a/src/test/core_model/test_phi4_core_backend.cpp b/src/test/core_model/test_phi4_core_backend.cpp index c9fb2c813..aa7e35eff 100644 --- a/src/test/core_model/test_phi4_core_backend.cpp +++ b/src/test/core_model/test_phi4_core_backend.cpp @@ -72,7 +72,7 @@ void TestTheCatalogsGgufRunsAsCorelibsPhi4mini() { auto backend = flm::phi4::core_factory()(Context(p, flm::phi4::kCoreContextLimit)); const auto& state = fake_corelib::GetState(); TEST_REQUIRE(state.model_names.front() == "phi4mini"); - TEST_REQUIRE(state.model_weights_path == (p.root / kGguf).string()); + TEST_REQUIRE(state.model_path == (p.root / kGguf).string()); TEST_REQUIRE(state.model_max_sequence_length == 4096); TEST_REQUIRE(backend->id() == flm::backend::kCoreBackendId); TEST_REQUIRE(backend->max_decode_length() == flm::phi4::kCoreDecodeLimit); @@ -87,7 +87,7 @@ void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { RequireContains(RequireThrows( [&] { (void)flm::phi4::core_factory()(Context(p, 4097)); }), "1..4096"); - TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_model_weights_create_path"] == 0); + TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_create_model_from_path"] == 0); } } // namespace diff --git a/src/test/phi4_core/fake_corelib.cpp b/src/test/phi4_core/fake_corelib.cpp index c9aed5018..be419f850 100644 --- a/src/test/phi4_core/fake_corelib.cpp +++ b/src/test/phi4_core/fake_corelib.cpp @@ -40,6 +40,10 @@ void* NewObject(std::string kind = "generic") { return object; } +bool IsKind(const void* object, std::string_view kind) { + return object && static_cast(object)->kind == kind; +} + /// \brief the size the fake claims a packed weight has /// \note Only has to be deterministic and descriptor-derived: corelib rejects /// a cached slice whose length is not exactly what the descriptor packs @@ -165,14 +169,8 @@ struct TypedFake { return current_detail.c_str(); } else if constexpr (std::is_same_v) { return state.selftest_status; - } else if constexpr (std::is_same_v) { - // 0.5.0 answers "is there an NPU" with the device pointer itself, - // NULL when there is none. The fake has no device to hand out, so - // any stable non-null address will do -- nothing dereferences it. - static const int kFakeDevice = 0; - return state.has_device_context - ? static_cast(&kFakeDevice) - : nullptr; + } else if constexpr (std::is_same_v) { + return state.npu_usable; } else if constexpr (std::is_same_v) { void* object = std::get<0>(arguments); if (object) { @@ -202,23 +200,26 @@ struct TypedFake { const char* model = std::get<1>(arguments); state.kernel_bundles.emplace_back(model ? model : ""); return Status(Tag::name); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { const auto status = Status(Tag::name); state.model_names.emplace_back(std::get<0>(arguments) ? std::get<0>(arguments) : ""); - state.model_weights_path = std::get<1>(arguments) ? std::get<1>(arguments) : ""; + state.model_path = std::get<1>(arguments) ? std::get<1>(arguments) : ""; auto* out = std::get<2>(arguments); - if (out) *out = status == ryzenai_corelib_status_success ? NewObject("model_weights") : nullptr; + if (out) *out = status == ryzenai_corelib_status_success ? NewObject("model") : nullptr; return status; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { + // A model and a session are both an object_ptr to the compiler, + // so the fake tells them apart where corelib would. + if (!IsKind(std::get<0>(arguments), "model")) return ryzenai_corelib_status_bad_argument; const auto status = Status(Tag::name); - state.model_names.emplace_back(std::get<0>(arguments) ? std::get<0>(arguments) : ""); - const auto* options = std::get<2>(arguments); - auto* out = std::get<3>(arguments); + const auto* options = std::get<1>(arguments); + auto* out = std::get<2>(arguments); state.model_max_sequence_length = options ? options->max_sequence_length : 0; - ++state.model_creates; - if (out) *out = status == ryzenai_corelib_status_success ? NewObject("model") : nullptr; + ++state.session_creates; + if (out) *out = status == ryzenai_corelib_status_success ? NewObject("session") : nullptr; return status; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { + if (!IsKind(std::get<0>(arguments), "session")) return ryzenai_corelib_status_bad_argument; auto* out = std::get<1>(arguments); if (out) { constexpr std::int64_t kPage = 4096; @@ -227,6 +228,7 @@ struct TypedFake { } return Status(Tag::name); } else if constexpr (std::is_same_v) { + if (!IsKind(std::get<0>(arguments), "session")) return ryzenai_corelib_status_bad_argument; const auto* options = std::get<6>(arguments); auto* out_logits = std::get<8>(arguments); // corelib refuses options whose size does not cover the struct @@ -243,13 +245,18 @@ struct TypedFake { static_cast(state.model_vocab)), 0.5f); return Status(Tag::name); + } else if constexpr (std::is_same_v) { + if (!IsKind(std::get<0>(arguments), "session")) return ryzenai_corelib_status_bad_argument; + return Status(Tag::name); } else if constexpr (std::is_same_v) { + if (!IsKind(std::get<0>(arguments), "session")) return ryzenai_corelib_status_bad_argument; const auto status = Status(Tag::name); ++state.state_saves; auto* out = std::get<1>(arguments); if (out) *out = status == ryzenai_corelib_status_success ? NewObject("snapshot") : nullptr; return status; } else if constexpr (std::is_same_v) { + if (!IsKind(std::get<0>(arguments), "session")) return ryzenai_corelib_status_bad_argument; ++state.state_restores; return Status(Tag::name); } else if constexpr (std::is_same_v) { @@ -376,12 +383,11 @@ struct TypedFake { } return status; } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v) { // corelib 0.9 reports shipped kernels instead of padding a shape. // A negative M is the test double for "every requested M ships // exactly". Argument 1 is the callback, - // argument 2 is its context, for all three of these entry points. + // argument 2 is its context, for both of these entry points. const auto status = Status(Tag::name); if (status == ryzenai_corelib_status_success && !state.pad_row_overrides.empty()) { // A test that overrides a padded extent gets an explicit grid @@ -408,18 +414,11 @@ struct TypedFake { static_cast(64), false, static_cast(0), static_cast(0)); } - } else if constexpr (std::is_same_v) { + } else { for (const auto m : shipped("ssmlp")) callback(ctx, 0, "fusion_mladf_ssmlp", m, static_cast(3072), static_cast(8192), static_cast(64)); - } else { - for (const auto m : shipped("mha")) - callback(ctx, 0, static_cast(24), - static_cast(8), m, static_cast(0), - static_cast(128), static_cast(4096), - static_cast(96), static_cast(0), - false, false); } } } else if (status == ryzenai_corelib_status_success) { @@ -433,20 +432,11 @@ struct TypedFake { static_cast(0), false, static_cast(0), static_cast(0)); - } else if constexpr (std::is_same_v) { + } else { callback(ctx, 0, "", static_cast(-1), static_cast(0), static_cast(0), static_cast(0)); - } else { - callback(ctx, 0, static_cast(-1), - static_cast(0), - static_cast(-1), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(0), false, false); } } } @@ -537,8 +527,7 @@ struct TypedFake { state.work_in_flight = false; return Status(Tag::name); } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v) { if (state.statuses.contains("test_observe_dispatch_concurrency")) { const int active = ++state.active_leases; int maximum = state.maximum_active_leases.load(); @@ -566,20 +555,9 @@ struct TypedFake { } } } - if constexpr (std::is_same_v) { - // Real corelib reads the rotary tables on the host and rejects - // anything but a host view for them. - for (void* table : {std::get<5>(arguments), std::get<6>(arguments)}) { - if (!table || static_cast(table)->kind != "host_view") { - state.detail = "handle is not a class RyzenAI::CoreLib::HostView"; - return ryzenai_corelib_status_bad_argument; - } - } - } fake_corelib::DispatchRecord record{}; record.thread_id = std::this_thread::get_id(); - record.kind = std::is_same_v ? "matmul" : - std::is_same_v ? "ssmlp" : "mha"; + record.kind = std::is_same_v ? "matmul" : "ssmlp"; record.stream = std::get<0>(arguments); // 0.5.0 removed the row count from every dispatch: M is the leading // extent of the operand that was bound. Read it back off the input @@ -587,13 +565,9 @@ struct TypedFake { if constexpr (std::is_same_v) { record.input = std::get<1>(arguments); record.output = std::get<3>(arguments); - } else if constexpr (std::is_same_v) { + } else { record.input = std::get<1>(arguments); record.output = std::get<5>(arguments); - } else { - record.input = std::get<2>(arguments); - record.position = std::get<4>(arguments); - record.output = std::get<9>(arguments); } if (record.input) { const auto& shape = static_cast(record.input)->shape; @@ -663,7 +637,7 @@ void Reset() { RYZENAI_CORELIB_VERSION_PATCH}; state.selftest_status = ryzenai_corelib_status_success; state.default_status = ryzenai_corelib_status_success; - state.has_device_context = true; + state.npu_usable = true; state.stream_prefill_pdis.clear(); state.stream_token_pdis.clear(); state.detail.clear(); @@ -681,7 +655,6 @@ void Reset() { state.maximum_active_leases = 0; state.matmul_pad_calls.clear(); state.rows_pad_calls.clear(); - state.mha_pad_calls.clear(); state.pad_multiple = 64; state.matmul_k_delta = 0; state.matmul_n_delta = 0; @@ -701,12 +674,12 @@ void Reset() { state.fail_after_submit.clear(); state.model_max_sequence_length = -1; state.model_vocab = 8; - state.model_creates = 0; + state.session_creates = 0; state.state_saves = 0; state.state_restores = 0; state.model_forwards.clear(); state.model_names.clear(); - state.model_weights_path.clear(); + state.model_path.clear(); } flm::corelib::CorelibApi::Resolver Resolver() { diff --git a/src/test/phi4_core/fake_corelib.hpp b/src/test/phi4_core/fake_corelib.hpp index 358bb04c1..e45f6a51f 100644 --- a/src/test/phi4_core/fake_corelib.hpp +++ b/src/test/phi4_core/fake_corelib.hpp @@ -28,11 +28,6 @@ struct RowsPadCall { std::uint32_t group_size; }; -struct MhaPadCall { - std::int64_t m; - ryzenai_corelib_flat_mha_bf16_desc desc; -}; - struct TensorCreateRecord { ryzenai_corelib_data_type data_type; std::vector shape; @@ -103,11 +98,8 @@ struct State { RYZENAI_CORELIB_VERSION_PATCH}; ryzenai_corelib_status selftest_status{ryzenai_corelib_status_success}; ryzenai_corelib_status default_status{ryzenai_corelib_status_success}; - /// \brief whether get_device() hands back a device rather than NULL - /// \note Still a bool here even though 0.5.0's entry point returns a - /// pointer: what a test wants to say is "this box has an NPU or it - /// does not", and the fake has no device object worth modelling. - bool has_device_context{true}; + /// \brief what npu_usable() answers + bool npu_usable{true}; /// \brief the PDI lists the engine opened its stream with /// \note Required from 0.5.0 and deliberately undefaulted by corelib, so /// it is worth being able to assert on. @@ -128,7 +120,6 @@ struct State { std::atomic maximum_active_leases{0}; std::vector matmul_pad_calls; std::vector rows_pad_calls; - std::vector mha_pad_calls; std::int64_t pad_multiple{64}; std::int64_t matmul_k_delta{0}; std::int64_t matmul_n_delta{0}; @@ -149,13 +140,13 @@ struct State { std::string fail_after_submit; std::int64_t model_max_sequence_length{-1}; std::int64_t model_vocab{8}; - std::size_t model_creates{}; + std::size_t session_creates{}; std::size_t state_saves{}; std::size_t state_restores{}; std::vector model_forwards; - /// \brief the name passed to model_weights_create_path, then to every model_create + /// \brief the name passed to each create_model_from_path std::vector model_names; - std::string model_weights_path; + std::string model_path; }; State& GetState(); diff --git a/src/test/phi4_core/test_corelib_api.cpp b/src/test/phi4_core/test_corelib_api.cpp index e58371368..431361405 100644 --- a/src/test/phi4_core/test_corelib_api.cpp +++ b/src/test/phi4_core/test_corelib_api.cpp @@ -214,7 +214,7 @@ void TestEveryUniqueObjectReleasesExactlyOnceAfterMoves() { TEST_REQUIRE(api->live_object_count() == 0); } -void TestRuntimeRunsDependencySelftestAndRequiresDeviceContext() { +void TestRuntimeRunsDependencySelftestAndRequiresAUsableNpu() { fake_corelib::Reset(); fake_corelib::GetState().selftest_status = ryzenai_corelib_status_failure; RequireContains(RequireThrows([] { @@ -223,11 +223,11 @@ void TestRuntimeRunsDependencySelftestAndRequiresDeviceContext() { "selftest_dependencies"); fake_corelib::Reset(); - fake_corelib::GetState().has_device_context = false; + fake_corelib::GetState().npu_usable = false; RequireContains(RequireThrows([] { CorelibRuntime::CreateForTest(ValidApi()); }), - "device context"); + "no usable NPU"); fake_corelib::Reset(); const auto runtime = CorelibRuntime::CreateForTest(ValidApi()); @@ -314,7 +314,7 @@ int main() { RUN_TEST(TestEnvironmentPathWinsOverExecutableRelativePath); RUN_TEST(TestFallbackIsExeDirectoryCoreDllNotCurrentDirectory); RUN_TEST(TestEveryUniqueObjectReleasesExactlyOnceAfterMoves); - RUN_TEST(TestRuntimeRunsDependencySelftestAndRequiresDeviceContext); + RUN_TEST(TestRuntimeRunsDependencySelftestAndRequiresAUsableNpu); RUN_TEST(TestExecutionLeaseSerializesTwoThreads); RUN_TEST(TestShutdownReleasesExecutionLockBeforeDestroyingRuntimeOwner); RUN_TEST(TestCleanupRunsAfterTheLastObjectAndOnlyOnce); From ca985e6cdb6576936fd9ff59f697cdcd0cabcc09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 19:16:53 -0700 Subject: [PATCH 05/13] feat(core): Qwen3.8-27B, Qwen3.6 at Q4_G32, each model to its paged maximum Qwen3.8-27B runs on corelib's qwen38 under the qwen3.8-mtp family, so it takes the Qwen3_8MTP frontend; insert() reached the engine's restore() and checkpoint() through a cast to the FastFlowLM engine, which is null for a core engine, and now calls them on the causal_lm, and the MTP loop speculates only where the engine says it can. Its catalog entry, qwen3.8-core:27b, prefers amd/Qwen3.8-27B-Q4_G128-GGUF, the Q4_G128 corelib runs on the NPU, converted from amd/Qwen3.8-27B_gpuep_1.0_gpu_128 at 9b37cf56 (15,964,921,824 bytes, the sha256 corelib's qwen38.md records), and falls back to unsloth's public Q4_0, which corelib runs on the GPU. Qwen3.6 prefers amd/Qwen3.6-35B-A3B-Q4_G32-GGUF over its public Q8_0. Both repositories are pinned to main until they are published; model_info.json carries every file's size and sha256. corelib 0.53 pages every model's KV cache a page at a time, so each backend takes the most corelib runs it at: Qwen3 32768, Llama, Phi-4-mini and Gemma 4 131072, Qwen3.5, Qwen3.6 and Qwen3.8 262144. Gemma 4's rule of whole pages past 4096 goes, corelib rounding any capacity up. Llama, Qwen3, Phi-4-mini and Gemma 4 continue from the page a later prompt starts in rather than from 0: corelib steps a pass off a page boundary one row at a time, and a whole page from a boundary runs as one. Qwen3.5 now restores a snapshot at its checkpoint instead of replaying. The catalog defaults to the longest context corelib records no accuracy problem at: Llama and Phi-4-mini 131072, Qwen3 32768, Gemma 4 65536 (PAGED_ATTENTION_POSITION_WRAP past 65535), and 32768 for Qwen3.5, Qwen3.6 and Qwen3.8, whose paged decode drifts the more pages it attends (PAGED_DECODE_PAGE_DRIFT); a longer one up to the limit is asked for. Co-authored-by: Cursor --- src/common/AutoModel/builtin_backends.cpp | 4 + src/common/AutoModel/modeling_qwen3_8mtp.cpp | 6 +- .../gemma4/core/dev04/gemma4_core_backend.cpp | 10 +- .../llama/core/dev04/llama_core_backend.cpp | 11 +- .../phi4/core/dev04/phi4_core_backend.cpp | 12 +- .../qwen3/core/dev04/qwen3_core_backend.cpp | 11 +- .../qwen35/core/dev04/qwen35_core_backend.cpp | 6 +- .../qwen38/core/dev04/qwen38_core_backend.cpp | 25 ++++ .../gemma4/core/dev04/gemma4_core_backend.hpp | 15 +-- .../llama/core/dev04/llama_core_backend.hpp | 3 +- .../phi4/core/dev04/phi4_core_backend.hpp | 4 +- .../qwen3/core/dev04/qwen3_core_backend.hpp | 3 +- .../qwen35/core/dev04/qwen35_core_backend.hpp | 5 +- .../qwen36/core/dev04/qwen36_core_backend.hpp | 8 +- .../qwen38/core/dev04/qwen38_core_backend.hpp | 26 ++++ src/model_info.json | 96 +++++++++++++++ src/model_list.json | 114 ++++++++++++++++-- .../core_model/test_llama_core_backend.cpp | 12 +- .../core_model/test_phi4_core_backend.cpp | 10 +- src/test/gemma4_core/test_gemma4_registry.cpp | 29 ++--- .../test_model_list_platform.cpp | 2 +- src/test/phi4_core/test_model_downloader.cpp | 3 +- src/test/phi4_core/test_phi4_frontend.cpp | 2 +- 23 files changed, 328 insertions(+), 89 deletions(-) create mode 100644 src/common/models/qwen38/core/dev04/qwen38_core_backend.cpp create mode 100644 src/include/models/qwen38/core/dev04/qwen38_core_backend.hpp diff --git a/src/common/AutoModel/builtin_backends.cpp b/src/common/AutoModel/builtin_backends.cpp index bdb17f1f3..6415b643f 100644 --- a/src/common/AutoModel/builtin_backends.cpp +++ b/src/common/AutoModel/builtin_backends.cpp @@ -15,6 +15,7 @@ #include "models/qwen3/core/dev04/qwen3_core_backend.hpp" #include "models/qwen35/core/dev04/qwen35_core_backend.hpp" #include "models/qwen36/core/dev04/qwen36_core_backend.hpp" +#include "models/qwen38/core/dev04/qwen38_core_backend.hpp" #endif namespace flm::backend { @@ -75,6 +76,9 @@ void register_builtin_backends(BackendRegistry& registry) { registry.register_backend("qwen3.6-moe", flm::backend::kCoreBackendId, flm::qwen36::core_factory(), flm::qwen36::core_traits()); + registry.register_backend("qwen3.8-mtp", flm::backend::kCoreBackendId, + flm::qwen38::core_factory(), + flm::qwen38::core_traits()); registry.register_backend("llama3", flm::backend::kCoreBackendId, flm::llama::core_factory(), flm::llama::core_traits()); diff --git a/src/common/AutoModel/modeling_qwen3_8mtp.cpp b/src/common/AutoModel/modeling_qwen3_8mtp.cpp index fea2e746c..08a0d7134 100644 --- a/src/common/AutoModel/modeling_qwen3_8mtp.cpp +++ b/src/common/AutoModel/modeling_qwen3_8mtp.cpp @@ -330,7 +330,7 @@ bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int restore_idx = -1; if (meta_info.restore_allowed) { - restore_idx = qwen3_8mtp_engine->restore(); + restore_idx = this->lm_engine->restore(); this->total_tokens = restore_idx; this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic } @@ -360,7 +360,7 @@ bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); checkpoint_his = token_history; - int checkpoint_idx = qwen3_8mtp_engine->checkpoint(); + int checkpoint_idx = this->lm_engine->checkpoint(); return success; } @@ -433,7 +433,7 @@ std::string Qwen3_8MTP::_speculative_generate(chat_meta_info_t& meta_info, int l // FLM_DISABLE_SPECULATION=1 turns it off too, for comparing against plain // decoding (output, logprobs) on the same build. const char* no_spec = std::getenv("FLM_DISABLE_SPECULATION"); - const bool spec_enabled = !meta_info.constraint && + const bool spec_enabled = this->lm_engine->supports_speculation() && !meta_info.constraint && !(no_spec != nullptr && no_spec[0] != '\0' && std::strcmp(no_spec, "0") != 0); // Draft depth; also the hit-rate denominator (see report_speculation_stats). const int SPEC_MAX_DRAFT = 7; diff --git a/src/common/models/gemma4/core/dev04/gemma4_core_backend.cpp b/src/common/models/gemma4/core/dev04/gemma4_core_backend.cpp index f17d05574..fd5de39a6 100644 --- a/src/common/models/gemma4/core/dev04/gemma4_core_backend.cpp +++ b/src/common/models/gemma4/core/dev04/gemma4_core_backend.cpp @@ -4,10 +4,13 @@ #include "core/corelib_model_backend.hpp" +#include + namespace flm::gemma4 { -/// \note A multi-row pass starts at 0 or, past 4096, on a page boundary, so a -/// later prompt re-runs its own page's earlier tokens with it. +/// \note corelib steps a pass that starts off a page boundary one row at a +/// time up to it, so a later prompt re-runs its own page's earlier tokens +/// with it and runs as whole pages. /// \note The package states its stop and start tokens by name: and /// in tokenizer_config.json, and config.json's eos_token_id [1, 106] /// adds . @@ -15,8 +18,7 @@ flm::backend::BackendFactory core_factory() { return core::Factory({ .model = {"gemma4", "Gemma 4", kCoreContextLimit, core::Continuation::kFromPage, core::Rewind::kSeek}, - .context_fits = ContextFits, - .context_rule = "1..4096, or a multiple of 4096 up to 16384", + .context_rule = "1.." + std::to_string(kCoreContextLimit), .read_bos = true, }); } diff --git a/src/common/models/llama/core/dev04/llama_core_backend.cpp b/src/common/models/llama/core/dev04/llama_core_backend.cpp index 4c84767ee..1bb63e183 100644 --- a/src/common/models/llama/core/dev04/llama_core_backend.cpp +++ b/src/common/models/llama/core/dev04/llama_core_backend.cpp @@ -4,15 +4,18 @@ #include "core/corelib_model_backend.hpp" +#include + namespace flm::llama { -/// \note A multi-row pass starts only at position 0, so a later prompt re-runs -/// the conversation with it. +/// \note corelib steps a pass that starts off a page boundary one row at a +/// time up to it, so a later prompt re-runs its own page's earlier tokens +/// with it and runs as whole pages. flm::backend::BackendFactory core_factory() { return core::Factory({ - .model = {"llama", "Llama", kCoreContextLimit, core::Continuation::kFromStart, + .model = {"llama", "Llama", kCoreContextLimit, core::Continuation::kFromPage, core::Rewind::kSeek}, - .context_rule = "1..4096", + .context_rule = "1.." + std::to_string(kCoreContextLimit), // <|eot_id|>, <|eom_id|>, <|end_of_text|>: eos_token_id in the // catalog's config.json .eos_ids = {128009, 128008, 128001}, diff --git a/src/common/models/phi4/core/dev04/phi4_core_backend.cpp b/src/common/models/phi4/core/dev04/phi4_core_backend.cpp index 68281e3d2..2c6983795 100644 --- a/src/common/models/phi4/core/dev04/phi4_core_backend.cpp +++ b/src/common/models/phi4/core/dev04/phi4_core_backend.cpp @@ -4,17 +4,19 @@ #include "core/corelib_model_backend.hpp" +#include + namespace flm::phi4 { /// \note corelib reads the catalog's GGUF, Q8_0 or Q4_G128, and checks its -/// shapes and metadata itself. A multi-row pass starts only at position -/// 0, so a later prompt re-runs the conversation with it, as the per-op -/// engine did. +/// shapes and metadata itself. It steps a pass that starts off a page +/// boundary one row at a time up to it, so a later prompt re-runs its own +/// page's earlier tokens with it and runs as whole pages. flm::backend::BackendFactory core_factory() { return core::Factory({ - .model = {"phi4mini", "Phi-4", kCoreContextLimit, core::Continuation::kFromStart, + .model = {"phi4mini", "Phi-4", kCoreContextLimit, core::Continuation::kFromPage, core::Rewind::kSeek}, - .context_rule = "1..4096", + .context_rule = "1.." + std::to_string(kCoreContextLimit), // <|end|>, <|endoftext|> .eos_ids = {200020, 199999}, }); diff --git a/src/common/models/qwen3/core/dev04/qwen3_core_backend.cpp b/src/common/models/qwen3/core/dev04/qwen3_core_backend.cpp index 501ae4940..d38c3c7fb 100644 --- a/src/common/models/qwen3/core/dev04/qwen3_core_backend.cpp +++ b/src/common/models/qwen3/core/dev04/qwen3_core_backend.cpp @@ -4,15 +4,18 @@ #include "core/corelib_model_backend.hpp" +#include + namespace flm::qwen3 { -/// \note A multi-row pass starts only at position 0, so a later prompt re-runs -/// the conversation with it, as the per-op engine did. +/// \note corelib steps a pass that starts off a page boundary one row at a +/// time up to it, so a later prompt re-runs its own page's earlier tokens +/// with it and runs as whole pages. flm::backend::BackendFactory core_factory() { return core::Factory({ - .model = {"qwen3", "Qwen3", kCoreContextLimit, core::Continuation::kFromStart, + .model = {"qwen3", "Qwen3", kCoreContextLimit, core::Continuation::kFromPage, core::Rewind::kSeek}, - .context_rule = "1..4096", + .context_rule = "1.." + std::to_string(kCoreContextLimit), // <|im_end|>, <|endoftext|> .eos_ids = {151645, 151643}, }); diff --git a/src/common/models/qwen35/core/dev04/qwen35_core_backend.cpp b/src/common/models/qwen35/core/dev04/qwen35_core_backend.cpp index fa4bd3bf0..7da069246 100644 --- a/src/common/models/qwen35/core/dev04/qwen35_core_backend.cpp +++ b/src/common/models/qwen35/core/dev04/qwen35_core_backend.cpp @@ -8,12 +8,12 @@ namespace flm::qwen35 { -/// \note The linear-attention state can neither rewind nor skip and corelib -/// keeps it outside any state slot, so going back means replaying. +/// \note The linear-attention state can neither rewind nor skip, so going back +/// is a snapshot restore at the checkpoint and a replay anywhere else. flm::backend::BackendFactory core_factory() { return core::Factory({ .model = {"qwen35", "Qwen3.5", kCoreContextLimit, core::Continuation::kAnywhere, - core::Rewind::kReplay}, + core::Rewind::kSnapshot}, .context_rule = "1.." + std::to_string(kCoreContextLimit), .eos_ids = {248046, 248044}, }); diff --git a/src/common/models/qwen38/core/dev04/qwen38_core_backend.cpp b/src/common/models/qwen38/core/dev04/qwen38_core_backend.cpp new file mode 100644 index 000000000..86032ee48 --- /dev/null +++ b/src/common/models/qwen38/core/dev04/qwen38_core_backend.cpp @@ -0,0 +1,25 @@ +/// \file qwen38_core_backend.cpp +/// \brief Qwen3.8-27B text on corelib's "qwen38" model +#include "models/qwen38/core/dev04/qwen38_core_backend.hpp" + +#include "core/corelib_model_backend.hpp" + +#include + +namespace flm::qwen38 { + +/// \note corelib runs it on the NPU only from the converted Q4_G128 export; +/// any other GGUF it takes runs on the GPU or the host. +/// \note The linear-attention state can neither rewind nor skip, so going back +/// is a snapshot restore at the checkpoint and a replay anywhere else. +flm::backend::BackendFactory core_factory() { + return core::Factory({ + .model = {"qwen38", "Qwen3.8", kCoreContextLimit, core::Continuation::kAnywhere, + core::Rewind::kSnapshot}, + .context_rule = "1.." + std::to_string(kCoreContextLimit), + // <|im_end|>, <|endoftext|> + .eos_ids = {248046, 248044}, + }); +} + +} // namespace flm::qwen38 diff --git a/src/include/models/gemma4/core/dev04/gemma4_core_backend.hpp b/src/include/models/gemma4/core/dev04/gemma4_core_backend.hpp index 91246a31b..60fd40639 100644 --- a/src/include/models/gemma4/core/dev04/gemma4_core_backend.hpp +++ b/src/include/models/gemma4/core/dev04/gemma4_core_backend.hpp @@ -9,23 +9,12 @@ namespace flm::gemma4 { -/// \brief rows in one page of the paged KV cache, and the longest unpaged context -inline constexpr std::uint32_t kPageSize = 4096; - -/// \brief the longest context validated on hardware -/// \note corelib pages Gemma 4 up to the model's own 131072 and refuses past a -/// GGUF's smaller gemma4.context_length; 16384 is what was validated. -inline constexpr std::uint32_t kCoreContextLimit = 16384; +/// \note corelib pages Gemma 4 up to the model's own context, at most 131072. +inline constexpr std::uint32_t kCoreContextLimit = 131072; /// \brief one below the longest context: the last position has to be readable inline constexpr std::uint32_t kCoreDecodeLimit = kCoreContextLimit - 1; -/// \return true when `length` is 1..4096, or a multiple of kPageSize up to kCoreContextLimit -inline bool ContextFits(std::uint32_t length) { - if (length < 1 || length > kCoreContextLimit) return false; - return length <= kPageSize || length % kPageSize == 0; -} - /// \note Inline: AutoModel reads it while assembling the BackendContext, /// before any engine exists. inline flm::backend::BackendTraits core_traits() { diff --git a/src/include/models/llama/core/dev04/llama_core_backend.hpp b/src/include/models/llama/core/dev04/llama_core_backend.hpp index d132089d0..b0b4668e8 100644 --- a/src/include/models/llama/core/dev04/llama_core_backend.hpp +++ b/src/include/models/llama/core/dev04/llama_core_backend.hpp @@ -10,7 +10,8 @@ namespace flm::llama { /// \brief the largest context this backend can hold -inline constexpr std::uint32_t kCoreContextLimit = 4096; +/// \note The most corelib pages Llama to, and the context of every size here. +inline constexpr std::uint32_t kCoreContextLimit = 131072; /// \brief what the frontend must know before building this backend /// \note Inline so the registry can reject a request before the engine exists. diff --git a/src/include/models/phi4/core/dev04/phi4_core_backend.hpp b/src/include/models/phi4/core/dev04/phi4_core_backend.hpp index 3b9478843..c87895b70 100644 --- a/src/include/models/phi4/core/dev04/phi4_core_backend.hpp +++ b/src/include/models/phi4/core/dev04/phi4_core_backend.hpp @@ -10,7 +10,9 @@ namespace flm::phi4 { /// \brief the largest context this backend can hold -inline constexpr std::uint32_t kCoreContextLimit = 4096; +/// \note The most corelib pages Phi-4-mini to; past one page it ropes every +/// position with LongRoPE's long factors. +inline constexpr std::uint32_t kCoreContextLimit = 131072; /// \brief the largest number of tokens corelib will decode inline constexpr std::uint32_t kCoreDecodeLimit = kCoreContextLimit - 1; diff --git a/src/include/models/qwen3/core/dev04/qwen3_core_backend.hpp b/src/include/models/qwen3/core/dev04/qwen3_core_backend.hpp index dfc366a25..440e2515d 100644 --- a/src/include/models/qwen3/core/dev04/qwen3_core_backend.hpp +++ b/src/include/models/qwen3/core/dev04/qwen3_core_backend.hpp @@ -10,7 +10,8 @@ namespace flm::qwen3 { /// \brief the largest context this backend can hold -inline constexpr std::uint32_t kCoreContextLimit = 4096; +/// \note Qwen3's native context, the most corelib pages it to. +inline constexpr std::uint32_t kCoreContextLimit = 32768; /// \brief what the frontend must know before building this backend /// \note Inline so the registry can reject a request before the engine exists. diff --git a/src/include/models/qwen35/core/dev04/qwen35_core_backend.hpp b/src/include/models/qwen35/core/dev04/qwen35_core_backend.hpp index c06534e17..3fe68f81e 100644 --- a/src/include/models/qwen35/core/dev04/qwen35_core_backend.hpp +++ b/src/include/models/qwen35/core/dev04/qwen35_core_backend.hpp @@ -9,8 +9,9 @@ namespace flm::qwen35 { -/// \note corelib rounds a context past 4096 up to whole 4096-row pages. -inline constexpr std::uint32_t kCoreContextLimit = 32768; +/// \note Qwen3.5's native context, the most corelib pages it to; corelib +/// rounds a context up to whole 4096-row pages. +inline constexpr std::uint32_t kCoreContextLimit = 262144; inline flm::backend::BackendTraits core_traits() { flm::backend::BackendTraits traits; diff --git a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp index 3e9c0bb1d..b5a1761d1 100644 --- a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp +++ b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp @@ -9,9 +9,11 @@ namespace flm::qwen36 { -/// \note corelib takes up to the model's own 262144; this stays at 131072, -/// the most FastFlowLM has run on the NPU. -inline constexpr std::uint32_t kCoreContextLimit = 131072; +/// \note The model's own context, the most corelib pages it to. Past 32768 +/// its NPU decode loses what lies far back (PAGED_DECODE_PAGE_DRIFT in +/// corelib's docs/known-problems.md), so the catalog defaults to 32768 +/// and a longer context has to be asked for. +inline constexpr std::uint32_t kCoreContextLimit = 262144; inline flm::backend::BackendTraits core_traits() { flm::backend::BackendTraits traits; diff --git a/src/include/models/qwen38/core/dev04/qwen38_core_backend.hpp b/src/include/models/qwen38/core/dev04/qwen38_core_backend.hpp new file mode 100644 index 000000000..ce83ea94b --- /dev/null +++ b/src/include/models/qwen38/core/dev04/qwen38_core_backend.hpp @@ -0,0 +1,26 @@ +/// \file qwen38_core_backend.hpp +/// \brief The ryzenai-corelib backend for Qwen3.8-27B text +/// \note Compiled only when FLM_ENABLE_CORE is on. +#pragma once + +#include "AutoModel/model_backend.hpp" + +#include + +namespace flm::qwen38 { + +/// \note The model's own context, the most corelib pages it to; corelib +/// rounds a context up to whole 4096-row pages. +inline constexpr std::uint32_t kCoreContextLimit = 262144; + +inline flm::backend::BackendTraits core_traits() { + flm::backend::BackendTraits traits; + traits.needs_npu_xclbin = false; + traits.supports_preemption = false; + traits.max_context_length = kCoreContextLimit; + return traits; +} + +flm::backend::BackendFactory core_factory(); + +} // namespace flm::qwen38 diff --git a/src/model_info.json b/src/model_info.json index 1d3f2c9bc..d55721d3a 100644 --- a/src/model_info.json +++ b/src/model_info.json @@ -4856,5 +4856,101 @@ "size": 1020, "sha256": "82ff79502694a38eacd424763c5f3cc8a95843dd233d830b74dee5b3a14dde9b" } + ], + "qwen3.6-moe-core:35b-a3b@amd": [ + { + "type": "file", + "path": "Qwen3.6-35B-A3B-Q4_G32.gguf", + "size": 21298897504, + "sha256": "f71ec12919fe53e9c273588e297e4b73f8ae3f1ece9b2607a7c11e44b7a6ecac" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 12807982, + "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 16718, + "sha256": "5186f0defcd7f232382c7f0aebcd2252d073bb921ab240e407b7ae8745d2b29b" + }, + { + "type": "file", + "path": "config.json", + "size": 3686, + "sha256": "93a4693fa9d8392fbfccd4b3c9873f4bfdcb14fdede978b123d07d19675efe99" + }, + { + "type": "file", + "path": "chat_template.jinja", + "size": 7764, + "sha256": "e84f32a23fdda27689f868aa4a1a5621f41133e51a48d7f3efcbea2839574259" + } + ], + "qwen3.8-core:27b": [ + { + "type": "file", + "path": "Qwen3.8-27B-Q4_0.gguf", + "size": 16056478688, + "sha256": "ede16c7b36e578ca87a8c70e011e4b4633a32c831c0ce76d0f474582384e671d" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 12809320, + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 17928, + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27" + }, + { + "type": "file", + "path": "config.json", + "size": 4312, + "sha256": "191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab" + }, + { + "type": "file", + "path": "chat_template.jinja", + "size": 8952, + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041" + } + ], + "qwen3.8-core:27b@amd": [ + { + "type": "file", + "path": "Qwen3.8-27B-Q4_G128.gguf", + "size": 15964921824, + "sha256": "31fae186a73095d2dc1b1e51847cf2f28af973ba0ff5982df00feaa278af27f0" + }, + { + "type": "file", + "path": "tokenizer.json", + "size": 12809320, + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3" + }, + { + "type": "file", + "path": "tokenizer_config.json", + "size": 17928, + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27" + }, + { + "type": "file", + "path": "config.json", + "size": 4312, + "sha256": "191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab" + }, + { + "type": "file", + "path": "chat_template.jinja", + "size": 8952, + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041" + } ] } diff --git a/src/model_list.json b/src/model_list.json index 182e671ad..529a5e79f 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -782,7 +782,7 @@ "url": "https://huggingface.co/unsloth/Phi-4-mini-instruct-GGUF/resolve/78eb92a46fc37e6b524df991ed9aca9bc6aa7b80", "file_url": "https://huggingface.co/api/models/unsloth/Phi-4-mini-instruct-GGUF/tree/78eb92a46fc37e6b524df991ed9aca9bc6aa7b80", "size": 4100140571, - "default_context_length": 4096, + "default_context_length": 131072, "max_prefill_len": 4096, "details": { "family": "phi4", @@ -1638,7 +1638,7 @@ "url": "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/0314792d7f1f7e229411f620751375812bb9faf2", "file_url": "https://huggingface.co/api/models/unsloth/gemma-4-E2B-it-GGUF/tree/0314792d7f1f7e229411f620751375812bb9faf2", "size": 5080530526, - "default_context_length": 16384, + "default_context_length": 65536, "max_prefill_len": 4096, "details": { "family": "gemma4e", @@ -1707,7 +1707,7 @@ "url": "https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/bfc15c382204943c3a8fff0c750b94ae2364d7a3", "file_url": "https://huggingface.co/api/models/unsloth/gemma-4-E4B-it-GGUF/tree/bfc15c382204943c3a8fff0c750b94ae2364d7a3", "size": 8225131325, - "default_context_length": 16384, + "default_context_length": 65536, "max_prefill_len": 4096, "details": { "family": "gemma4e", @@ -1778,7 +1778,7 @@ "url": "https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/e87f176479d0855a907a41277aca2f8ee7a09523", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3.5-4B-GGUF/tree/e87f176479d0855a907a41277aca2f8ee7a09523", "size": 4495231341, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3.5", @@ -1840,7 +1840,7 @@ "url": "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3.5-9B-GGUF/tree/3885219b6810b007914f3a7950a8d1b469d598a5", "size": 9540331866, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3.5", @@ -1904,7 +1904,7 @@ "url": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/a483e9e6cbd595906af30beda3187c2663a1118c", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3.6-35B-A3B-GGUF/tree/a483e9e6cbd595906af30beda3187c2663a1118c", "size": 36915976470, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3.6-moe", @@ -1939,6 +1939,24 @@ "revision": "995ad96eacd98c81ed38be0c5b274b04031597b0" } }, + "preferred_source": { + "url": "https://huggingface.co/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/resolve/main", + "file_url": "https://huggingface.co/api/models/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/tree/main", + "use_hf_token": true, + "size": 21311733654, + "footprint": 21.3, + "details": { + "quantization_level": "Q4_G32" + }, + "files": [ + "Qwen3.6-35B-A3B-Q4_G32.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json", + "chat_template.jinja" + ], + "model_info_key": "qwen3.6-moe-core:35b-a3b@amd" + }, "footprint": 36.9, "model_info_key": "qwen3.6-moe-core:35b-a3b", "label": [ @@ -1946,6 +1964,76 @@ "tool-calling" ] } + }, + "qwen3.8-core": { + "27b": { + "supported_platforms": [ + "dev04" + ], + "name": "Qwen3.8-27B", + "url": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502", + "file_url": "https://huggingface.co/api/models/unsloth/Qwen3.8-27B-GGUF/tree/4ca720788d1e01f1bff70c033e0d0028fd02e502", + "size": 16069319200, + "default_context_length": 32768, + "max_prefill_len": 4096, + "details": { + "family": "qwen3.8-mtp", + "think": true, + "parameter_size": "27B", + "quantization_level": "Q4_0, runs on the GPU" + }, + "flm_min_version": "1.0.11", + "vlm": false, + "files": [ + "Qwen3.8-27B-Q4_0.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json", + "chat_template.jinja" + ], + "file_sources": { + "tokenizer.json": { + "url": "https://huggingface.co/Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + }, + "tokenizer_config.json": { + "url": "https://huggingface.co/Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + }, + "config.json": { + "url": "https://huggingface.co/Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + }, + "chat_template.jinja": { + "url": "https://huggingface.co/Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + } + }, + "preferred_source": { + "url": "https://huggingface.co/amd/Qwen3.8-27B-Q4_G128-GGUF/resolve/main", + "file_url": "https://huggingface.co/api/models/amd/Qwen3.8-27B-Q4_G128-GGUF/tree/main", + "use_hf_token": true, + "size": 15977762336, + "footprint": 15.96, + "details": { + "quantization_level": "Q4_G128" + }, + "files": [ + "Qwen3.8-27B-Q4_G128.gguf", + "tokenizer.json", + "tokenizer_config.json", + "config.json", + "chat_template.jinja" + ], + "model_info_key": "qwen3.8-core:27b@amd" + }, + "footprint": 16.06, + "model_info_key": "qwen3.8-core:27b", + "label": [ + "reasoning", + "tool-calling" + ] + } }, "qwen3-core": { "0.6b": { @@ -1956,7 +2044,7 @@ "url": "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/50968a4468ef4233ed78cd7c3de230dd1d61a56b", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3-0.6B-GGUF/tree/50968a4468ef4233ed78cd7c3de230dd1d61a56b", "size": 650881322, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3", @@ -1998,7 +2086,7 @@ "url": "https://huggingface.co/unsloth/Qwen3-1.7B-GGUF/resolve/d7f544eead698dbd1f15126ef60b45a1e1933222", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3-1.7B-GGUF/tree/d7f544eead698dbd1f15126ef60b45a1e1933222", "size": 1845860522, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3", @@ -2057,7 +2145,7 @@ "url": "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/22c9fc8a8c7700b76a1789366280a6a5a1ad1120", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3-4B-GGUF/tree/22c9fc8a8c7700b76a1789366280a6a5a1ad1120", "size": 4291839353, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3", @@ -2116,7 +2204,7 @@ "url": "https://huggingface.co/unsloth/Qwen3-8B-GGUF/resolve/a6adef130ffb23ddaf1a62fec9dced968c9bc482", "file_url": "https://huggingface.co/api/models/unsloth/Qwen3-8B-GGUF/tree/a6adef130ffb23ddaf1a62fec9dced968c9bc482", "size": 8720952748, - "default_context_length": 4096, + "default_context_length": 32768, "max_prefill_len": 4096, "details": { "family": "qwen3", @@ -2175,7 +2263,7 @@ "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/067b946cf014b7c697f3654f621d577a3e3afd1c", "file_url": "https://huggingface.co/api/models/bartowski/Llama-3.2-1B-Instruct-GGUF/tree/067b946cf014b7c697f3654f621d577a3e3afd1c", "size": 1330174015, - "default_context_length": 4096, + "default_context_length": 131072, "max_prefill_len": 4096, "details": { "family": "llama3", @@ -2214,7 +2302,7 @@ "url": "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/5ab33fa94d1d04e903623ae72c95d1696f09f9e8", "file_url": "https://huggingface.co/api/models/bartowski/Llama-3.2-3B-Instruct-GGUF/tree/5ab33fa94d1d04e903623ae72c95d1696f09f9e8", "size": 3430990303, - "default_context_length": 4096, + "default_context_length": 131072, "max_prefill_len": 4096, "details": { "family": "llama3", @@ -2255,7 +2343,7 @@ "url": "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/bf5b95e96dac0462e2a09145ec66cae9a3f12067", "file_url": "https://huggingface.co/api/models/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/tree/bf5b95e96dac0462e2a09145ec66cae9a3f12067", "size": 8549866848, - "default_context_length": 4096, + "default_context_length": 131072, "max_prefill_len": 4096, "details": { "family": "llama3", diff --git a/src/test/core_model/test_llama_core_backend.cpp b/src/test/core_model/test_llama_core_backend.cpp index f00ebd524..cfe5f8782 100644 --- a/src/test/core_model/test_llama_core_backend.cpp +++ b/src/test/core_model/test_llama_core_backend.cpp @@ -74,9 +74,9 @@ void TestTheCatalogsGgufRunsAsCorelibsLlama() { const auto& state = fake_corelib::GetState(); TEST_REQUIRE(state.model_names.front() == "llama"); TEST_REQUIRE(state.model_path == (p.root / kGguf).string()); - TEST_REQUIRE(state.model_max_sequence_length == 4096); + TEST_REQUIRE(state.model_max_sequence_length == flm::llama::kCoreContextLimit); TEST_REQUIRE(backend->id() == flm::backend::kCoreBackendId); - TEST_REQUIRE(backend->max_decode_length() == 4095); + TEST_REQUIRE(backend->max_decode_length() == flm::llama::kCoreContextLimit - 1); TEST_REQUIRE(!backend->supports_preemption()); TEST_REQUIRE(!backend->forwards_past_eos()); // <|eot_id|>, <|eom_id|>, <|end_of_text|> @@ -84,12 +84,12 @@ void TestTheCatalogsGgufRunsAsCorelibsLlama() { TEST_REQUIRE(!backend->forced_bos_id().has_value()); } -void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { +void TestAContextPastTheLimitIsRefusedBeforeTheWeightsAreOpened() { Runtime r; Package p; RequireContains(RequireThrows( - [&] { (void)flm::llama::core_factory()(Context(p, 4097)); }), - "1..4096"); + [&] { (void)flm::llama::core_factory()(Context(p, flm::llama::kCoreContextLimit + 1)); }), + "1..131072"); TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_create_model_from_path"] == 0); } @@ -98,7 +98,7 @@ void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { int main() { #define RUN(name) RunTest(&name, #name) RUN(TestTheCatalogsGgufRunsAsCorelibsLlama); - RUN(TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened); + RUN(TestAContextPastTheLimitIsRefusedBeforeTheWeightsAreOpened); #undef RUN std::cout << "All Llama core backend tests passed\n"; return 0; diff --git a/src/test/core_model/test_phi4_core_backend.cpp b/src/test/core_model/test_phi4_core_backend.cpp index aa7e35eff..95bd34f77 100644 --- a/src/test/core_model/test_phi4_core_backend.cpp +++ b/src/test/core_model/test_phi4_core_backend.cpp @@ -73,7 +73,7 @@ void TestTheCatalogsGgufRunsAsCorelibsPhi4mini() { const auto& state = fake_corelib::GetState(); TEST_REQUIRE(state.model_names.front() == "phi4mini"); TEST_REQUIRE(state.model_path == (p.root / kGguf).string()); - TEST_REQUIRE(state.model_max_sequence_length == 4096); + TEST_REQUIRE(state.model_max_sequence_length == flm::phi4::kCoreContextLimit); TEST_REQUIRE(backend->id() == flm::backend::kCoreBackendId); TEST_REQUIRE(backend->max_decode_length() == flm::phi4::kCoreDecodeLimit); TEST_REQUIRE(!backend->supports_preemption()); @@ -81,12 +81,12 @@ void TestTheCatalogsGgufRunsAsCorelibsPhi4mini() { TEST_REQUIRE(!backend->forced_bos_id().has_value()); } -void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { +void TestAContextPastTheLimitIsRefusedBeforeTheWeightsAreOpened() { Runtime r; Package p; RequireContains(RequireThrows( - [&] { (void)flm::phi4::core_factory()(Context(p, 4097)); }), - "1..4096"); + [&] { (void)flm::phi4::core_factory()(Context(p, flm::phi4::kCoreContextLimit + 1)); }), + "1..131072"); TEST_REQUIRE(fake_corelib::GetState().call_counts["ryzenai_corelib_create_model_from_path"] == 0); } @@ -95,7 +95,7 @@ void TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened() { int main() { #define RUN(name) RunTest(&name, #name) RUN(TestTheCatalogsGgufRunsAsCorelibsPhi4mini); - RUN(TestAContextPast4096IsRefusedBeforeTheWeightsAreOpened); + RUN(TestAContextPastTheLimitIsRefusedBeforeTheWeightsAreOpened); #undef RUN std::cout << "All Phi-4 core backend tests passed\n"; return 0; diff --git a/src/test/gemma4_core/test_gemma4_registry.cpp b/src/test/gemma4_core/test_gemma4_registry.cpp index 9ba71fe74..8dcf7cc9a 100644 --- a/src/test/gemma4_core/test_gemma4_registry.cpp +++ b/src/test/gemma4_core/test_gemma4_registry.cpp @@ -123,7 +123,7 @@ void TestTraitsAreReadableWithoutConstructingAnEngine() { const auto traits = flm::gemma4::core_traits(); TEST_REQUIRE(!traits.needs_npu_xclbin); TEST_REQUIRE(!traits.supports_preemption); - TEST_REQUIRE(traits.max_context_length == 16384); + TEST_REQUIRE(traits.max_context_length == flm::gemma4::kCoreContextLimit); // Nothing above touched corelib, a GGUF or the filesystem, which is the // property that lets the frontend consult it while it is still ASSEMBLING // the BackendContext -- before any engine exists to ask. @@ -140,21 +140,11 @@ void TestTraitsComeBackOutOfTheRegistryUnchanged() { void TestTheTraitsCeilingIsTheEnginesOwnLimit() { // The frontend rejects an over-long context against the TRAITS. The - // engine accepts the same ceiling: 1..4096 unpaged, or whole pages up to - // kCoreContextLimit. - TEST_REQUIRE(flm::gemma4::kCoreContextLimit == 16384); + // engine accepts the same ceiling, the most corelib pages Gemma 4 to. + TEST_REQUIRE(flm::gemma4::kCoreContextLimit == 131072); TEST_REQUIRE(flm::gemma4::kCoreDecodeLimit + 1 == flm::gemma4::kCoreContextLimit); } -void TestContextFitsStopsAtTheValidated16384() { - TEST_REQUIRE(flm::gemma4::ContextFits(4096)); - TEST_REQUIRE(flm::gemma4::ContextFits(8192)); - TEST_REQUIRE(flm::gemma4::ContextFits(16384)); - TEST_REQUIRE(!flm::gemma4::ContextFits(10000)); - TEST_REQUIRE(!flm::gemma4::ContextFits(20480)); - TEST_REQUIRE(!flm::gemma4::ContextFits(131072)); -} - void TestTheFactoryReadsTheShippedPackagesTokensAndContextRule() { TempPackage package; FakeRuntimeScope runtime; @@ -168,10 +158,15 @@ void TestTheFactoryReadsTheShippedPackagesTokensAndContextRule() { TEST_REQUIRE(backend->max_decode_length() == 8192 - 1); } TEST_REQUIRE(runtime.nothing_created()); - Request refused(package, 5000); + { + Request off_page(package, 5000); + auto backend = flm::gemma4::core_factory()(off_page.context); + TEST_REQUIRE(backend->max_decode_length() == 5000 - 1); + } + Request refused(package, flm::gemma4::kCoreContextLimit + 1); RequireContains(RequireThrows( [&] { (void)flm::gemma4::core_factory()(refused.context); }), - "multiple of 4096"); + "1..131072"); TEST_REQUIRE(runtime.nothing_created()); } @@ -340,8 +335,7 @@ void TestTripwireOneTheDev04FallbackIsGemma4AndItsFamilyCanLoad() { const auto traits = registry.traits(family, kCore); const auto default_context = info.at("default_context_length").get(); - TEST_REQUIRE(default_context <= traits.max_context_length); - TEST_REQUIRE(flm::gemma4::ContextFits(default_context)); + TEST_REQUIRE(default_context >= 1 && default_context <= traits.max_context_length); } void TestTripwireOneAnUnresolvableTagStillLoadsRatherThanThrowing() { @@ -588,7 +582,6 @@ int main() { RUN_TEST(TestTraitsAreReadableWithoutConstructingAnEngine); RUN_TEST(TestTraitsComeBackOutOfTheRegistryUnchanged); RUN_TEST(TestTheTraitsCeilingIsTheEnginesOwnLimit); - RUN_TEST(TestContextFitsStopsAtTheValidated16384); RUN_TEST(TestTheFactoryReadsTheShippedPackagesTokensAndContextRule); RUN_TEST(TestFunctionBodyStopsAtTheFunctionItNames); diff --git a/src/test/model_list_platform/test_model_list_platform.cpp b/src/test/model_list_platform/test_model_list_platform.cpp index 479c018e1..9e21a59a6 100644 --- a/src/test/model_list_platform/test_model_list_platform.cpp +++ b/src/test/model_list_platform/test_model_list_platform.cpp @@ -92,7 +92,7 @@ void test_a_corelib_build_offers_the_corelib_phi4() { TEST_REQUIRE(tag == kCoreTag); TEST_REQUIRE(corelib.rectify_model_tag("phi4-mini-it-core") == kCoreTag); TEST_REQUIRE(info.at("name") == "phi4-mini-it-core"); - TEST_REQUIRE(info.at("default_context_length") == 4096); + TEST_REQUIRE(info.at("default_context_length") == 131072); TEST_REQUIRE(info.at("flm_min_version") == "1.0.3"); TEST_REQUIRE(info.at("files").size() == 4); TEST_REQUIRE(info.at("file_sources").size() == 3); diff --git a/src/test/phi4_core/test_model_downloader.cpp b/src/test/phi4_core/test_model_downloader.cpp index 39bfdbbeb..10bde8624 100644 --- a/src/test/phi4_core/test_model_downloader.cpp +++ b/src/test/phi4_core/test_model_downloader.cpp @@ -456,7 +456,8 @@ void TestShippedPreferredSourcesArePinnedAmdGgufs() { } TEST_REQUIRE(tags == std::set({ "phi4-mini-it-core:4b", "qwen3-core:1.7b", "qwen3-core:4b", "qwen3-core:8b", - "qwen3.5-core:4b", "qwen3.5-core:9b", "gemma4-it-core:e2b", "gemma4-it-core:e4b"})); + "qwen3.5-core:4b", "qwen3.5-core:9b", "qwen3.6-moe-core:35b-a3b", "qwen3.8-core:27b", + "gemma4-it-core:e2b", "gemma4-it-core:e4b"})); } const std::vector kBaseFiles = {"config.json", "base.gguf"}; diff --git a/src/test/phi4_core/test_phi4_frontend.cpp b/src/test/phi4_core/test_phi4_frontend.cpp index 605a477a6..6c3807103 100644 --- a/src/test/phi4_core/test_phi4_frontend.cpp +++ b/src/test/phi4_core/test_phi4_frontend.cpp @@ -168,7 +168,7 @@ std::unique_ptr MakeCoreStub( auto backend = std::make_unique( std::string(flm::backend::kCoreBackendId), context.context_length); - backend->decode_limit_ = flm::phi4::kCoreDecodeLimit; + backend->decode_limit_ = context.context_length - 1; backend->supports_preemption_ = false; backend->forwards_past_eos_ = false; backend->forced_eos_ids_ = std::vector({200020, 199999}); From 21f8a45fd8718fbe6941020806a933a336f0a5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 20:51:37 -0700 Subject: [PATCH 06/13] fix(core): retry a failed corelib pass from the start; warn past Qwen3.6's 32768 corelib fails only the call that failed, and a 4096-row paged prefill can intermittently fail to complete (PAGED_PREFILL_PREEMPTION_TIMEOUT): on an AIE4 test machine, with another user's process on the NPU, Qwen3.8 and Qwen3-4B lost a 19k-token prompt that ran clean on the next try, and corelib's own runner failed the same way. The engine was poisoned on the first failure, so every later request failed until a reload. CorelibModel now resets the session and runs everything it held plus the new tokens again from position 0, once, and poisons the engine only when that fails too or the failure is not corelib's. Qwen3.6's paged attention past 32768 can return garbage (a passkey at 48k tokens came back as "!" repeated, on an AIE4 test machine), so a context past 32768 still loads but says so: BackendSpec gains reliable_context and the reason given when a context exceeds it. Co-authored-by: Cursor --- src/common/core/corelib_model.cpp | 44 ++++++++++++++----- src/common/core/corelib_model_backend.cpp | 4 ++ src/common/models/README.md | 8 ++-- .../qwen36/core/dev04/qwen36_core_backend.cpp | 3 ++ src/include/core/corelib_model_backend.hpp | 5 +++ .../qwen36/core/dev04/qwen36_core_backend.hpp | 11 +++-- src/test/core_model/test_corelib_model.cpp | 39 ++++++++++++++++ .../core_model/test_corelib_model_backend.cpp | 30 +++++++++++++ src/test/phi4_core/fake_corelib.cpp | 6 +++ src/test/phi4_core/fake_corelib.hpp | 3 ++ 10 files changed, 136 insertions(+), 17 deletions(-) diff --git a/src/common/core/corelib_model.cpp b/src/common/core/corelib_model.cpp index 7021ab09b..e010988b6 100644 --- a/src/common/core/corelib_model.cpp +++ b/src/common/core/corelib_model.cpp @@ -166,10 +166,39 @@ struct CorelibModel::Impl { std::vector rows(history.begin() + static_cast(from), history.end()); rows.insert(rows.end(), fresh.begin(), fresh.end()); - Dispatch(rows, from, want_logits); + try { + Dispatch(rows, from, want_logits); + } catch (const corelib::CorelibError&) { + RunAgainFromStart(fresh, want_logits); + return; + } catch (...) { + Poison(); + throw; + } history.insert(history.end(), fresh.begin(), fresh.end()); } + /// \brief after a failed pass, reset and run everything the model held plus + /// `fresh` again from position 0, once + /// \note corelib fails only the call that failed, and some failures are + /// intermittent (PAGED_PREFILL_PREEMPTION_TIMEOUT in corelib's + /// docs/known-problems.md); a second failure poisons the engine. + void RunAgainFromStart(std::span fresh, bool want_logits) { + std::vector all(history.begin(), history.end()); + all.insert(all.end(), fresh.begin(), fresh.end()); + std::cout << "[FLM] " << spec.display_name << ": a corelib call failed; running the " + << all.size() << " tokens again from the start" << std::endl; + try { + api->Check(api->functions().model_reset(session.get()), "ryzenai_corelib_model_reset"); + history.clear(); + Dispatch(all, 0, want_logits); + } catch (...) { + Poison(); + throw; + } + history.assign(all.begin(), all.end()); + } + /// \brief where a pass adding `rows` tokens at position `at` has to start std::size_t PassStart(std::size_t at, std::size_t rows) const { if (at == 0 || rows == 1) return at; @@ -191,15 +220,10 @@ struct CorelibModel::Impl { spec.rewind == Rewind::kSeek ? from : RYZENAI_CORELIB_POSITION_CONTINUE; float* out_logits = want_logits ? logits.data() : nullptr; const std::size_t logits_capacity = want_logits ? logits.size() : 0; - try { - api->Check(api->functions().model_forward(session.get(), rows.data(), nullptr, - rows.size(), nullptr, 0, &options, - nullptr, out_logits, logits_capacity), - "ryzenai_corelib_model_forward"); - } catch (...) { - Poison(); - throw; - } + api->Check(api->functions().model_forward(session.get(), rows.data(), nullptr, rows.size(), + nullptr, 0, &options, nullptr, out_logits, + logits_capacity), + "ryzenai_corelib_model_forward"); } /// \brief make `target` the sequence the model holds, as cheaply as its state allows diff --git a/src/common/core/corelib_model_backend.cpp b/src/common/core/corelib_model_backend.cpp index 665c43c23..033a236d5 100644 --- a/src/common/core/corelib_model_backend.cpp +++ b/src/common/core/corelib_model_backend.cpp @@ -154,6 +154,10 @@ class CorelibBackend final : public flm::backend::ModelBackend { : length >= 1 && length <= spec.model.context_limit; if (!fits) throw std::out_of_range(name + " core context length must be in " + spec.context_rule); + if (spec.reliable_context && length > spec.reliable_context) + header_print("WARNING", name << " core context " << length << " is past " + << spec.reliable_context << ": " + << spec.unreliable_reason); context_length_ = length; auto tokens = ReadPackageTokens(spec, context); diff --git a/src/common/models/README.md b/src/common/models/README.md index f382c69e1..17cff6aab 100644 --- a/src/common/models/README.md +++ b/src/common/models/README.md @@ -219,9 +219,11 @@ engine type, never through a `causal_lm*`. (`kRequantizeThreads`) goes to corelib as `fast_packer_threads`: 0 lets corelib choose (8 threads), 1 keeps each create serial. The two multiply, so pick them together. See `phi4_core_constants.hpp` at `f827ca9a`. -- Expose `bool poisoned() const noexcept`. A corelib failure mid-decode usually - leaves device state that only a reload can clear; the backend surfaces this and - `AutoModel` turns it into a 500 that asks for an unload/reload. +- Expose `bool poisoned() const noexcept`. corelib fails only the call that + failed, so `CorelibModel` resets the session and runs everything it held again + from position 0, once; a second failure poisons the engine, the backend + surfaces this, and `AutoModel` turns it into a 500 that asks for an + unload/reload. - Put every magic number in a `_core_constants.hpp` with a comment on where it came from. diff --git a/src/common/models/qwen36/core/dev04/qwen36_core_backend.cpp b/src/common/models/qwen36/core/dev04/qwen36_core_backend.cpp index eced91417..20d7f2f39 100644 --- a/src/common/models/qwen36/core/dev04/qwen36_core_backend.cpp +++ b/src/common/models/qwen36/core/dev04/qwen36_core_backend.cpp @@ -13,6 +13,9 @@ flm::backend::BackendFactory core_factory() { .model = {"qwen36", "Qwen3.6", kCoreContextLimit, core::Continuation::kAnywhere, core::Rewind::kSnapshot}, .context_rule = "1.." + std::to_string(kCoreContextLimit), + .reliable_context = kReliableContext, + .unreliable_reason = "past it the NPU's paged attention can return garbage " + "(corelib's PAGED_DECODE_PAGE_DRIFT)", }); } diff --git a/src/include/core/corelib_model_backend.hpp b/src/include/core/corelib_model_backend.hpp index 976c289c2..a7836e501 100644 --- a/src/include/core/corelib_model_backend.hpp +++ b/src/include/core/corelib_model_backend.hpp @@ -20,6 +20,11 @@ struct BackendSpec { std::function context_fits; /// \brief the contexts allowed, as a refusal states them, e.g. "1..4096" std::string context_rule; + /// \brief the longest context the model is known to answer correctly at, + /// 0 for its whole range; a longer one loads with a warning + std::uint32_t reliable_context = 0; + /// \brief what goes wrong past reliable_context, for that warning + std::string unreliable_reason; /// \brief the ids generation stops on; empty reads them from the package: /// tokenizer_config.json's eos_token, then config.json's eos_token_id std::vector eos_ids; diff --git a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp index b5a1761d1..e3bb95ac7 100644 --- a/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp +++ b/src/include/models/qwen36/core/dev04/qwen36_core_backend.hpp @@ -9,12 +9,15 @@ namespace flm::qwen36 { -/// \note The model's own context, the most corelib pages it to. Past 32768 -/// its NPU decode loses what lies far back (PAGED_DECODE_PAGE_DRIFT in -/// corelib's docs/known-problems.md), so the catalog defaults to 32768 -/// and a longer context has to be asked for. +/// \note The model's own context, the most corelib pages it to. inline constexpr std::uint32_t kCoreContextLimit = 262144; +/// \brief the longest context it answers correctly at +/// \note Past it the NPU's paged attention loses what lies far back +/// (PAGED_DECODE_PAGE_DRIFT in corelib's docs/known-problems.md), so the +/// catalog defaults to it and a longer context loads with a warning. +inline constexpr std::uint32_t kReliableContext = 32768; + inline flm::backend::BackendTraits core_traits() { flm::backend::BackendTraits traits; traits.needs_npu_xclbin = false; diff --git a/src/test/core_model/test_corelib_model.cpp b/src/test/core_model/test_corelib_model.cpp index f1c23355c..e2ae25238 100644 --- a/src/test/core_model/test_corelib_model.cpp +++ b/src/test/core_model/test_corelib_model.cpp @@ -221,6 +221,42 @@ void TestAFailedForwardPoisonsTheEngineAndDropsTheCheckpoint() { TEST_REQUIRE(S().state_restores == 0); } +void TestAForwardThatFailsOnceRunsAgainFromTheStart() { + Harness h(kFromPage, 16384); + auto ids = Ids(5000); + (void)h.engine->prefill(ids); + const int resets = Resets(); + S().fail_counts["ryzenai_corelib_model_forward"] = 1; + std::vector more{1, 2, 3}; + (void)h.engine->prefill(more); + TEST_REQUIRE(!h.engine->poisoned()); + TEST_REQUIRE(Resets() == resets + 1); + TEST_REQUIRE(Last().position == 0 && Last().rows == 5003 && Last().logits); + TEST_REQUIRE(h.engine->get_current_context_length() == 5003); + (void)h.engine->forward(4); + TEST_REQUIRE(Last().position == 5003 && Last().rows == 1); +} + +void TestARecurrentModelRunsAgainFromTheStartAfterAFailure() { + Harness h(kSnapshot, 4096); + auto ids = Ids(10); + (void)h.engine->prefill(ids); + S().fail_counts["ryzenai_corelib_model_forward"] = 1; + (void)h.engine->forward(1); + TEST_REQUIRE(!h.engine->poisoned()); + TEST_REQUIRE(Last().rows == 11 && Last().position == RYZENAI_CORELIB_POSITION_CONTINUE); + TEST_REQUIRE(h.engine->get_current_context_length() == 11); +} + +void TestAForwardThatFailsTwicePoisonsTheEngine() { + Harness h(kFromPage, 4096); + auto ids = Ids(10); + (void)h.engine->prefill(ids); + S().fail_counts["ryzenai_corelib_model_forward"] = 2; + (void)RequireThrows([&] { (void)h.engine->forward(1); }); + TEST_REQUIRE(h.engine->poisoned()); +} + void TestCheckpointingTheSameStateTwiceSavesOnce() { Harness h(kSnapshot, 4096); auto ids = Ids(10); @@ -458,6 +494,9 @@ int main() { RUN(TestRewindingOntoADivergedCheckpointReplays); RUN(TestAFailedRestorePoisonsTheEngine); RUN(TestAFailedForwardPoisonsTheEngineAndDropsTheCheckpoint); + RUN(TestAForwardThatFailsOnceRunsAgainFromTheStart); + RUN(TestARecurrentModelRunsAgainFromTheStartAfterAFailure); + RUN(TestAForwardThatFailsTwicePoisonsTheEngine); RUN(TestCheckpointingTheSameStateTwiceSavesOnce); RUN(TestRestoringRightAfterACheckpointLoadsNothing); RUN(TestShrinkingBelowTheCheckpointDropsIt); diff --git a/src/test/core_model/test_corelib_model_backend.cpp b/src/test/core_model/test_corelib_model_backend.cpp index 9653625d6..4d705fba9 100644 --- a/src/test/core_model/test_corelib_model_backend.cpp +++ b/src/test/core_model/test_corelib_model_backend.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -118,6 +119,34 @@ void TestACustomContextRuleIsApplied() { TEST_REQUIRE(fake_corelib::GetState().model_max_sequence_length == 8192); } +/// \brief what `run` prints to std::cout +template +std::string CaptureStdout(F run) { + std::ostringstream captured; + auto* previous = std::cout.rdbuf(captured.rdbuf()); + try { + run(); + } catch (...) { + std::cout.rdbuf(previous); + throw; + } + std::cout.rdbuf(previous); + return captured.str(); +} + +void TestAContextPastTheReliableOneLoadsWithAWarning() { + Runtime r; + Package p({"m.gguf"}); + auto spec = Spec(); + spec.reliable_context = 2048; + spec.unreliable_reason = "the answers drift"; + const auto within = CaptureStdout([&] { (void)flm::core::Factory(spec)(Context(p, 2048)); }); + TEST_REQUIRE(within.find("WARNING") == std::string::npos); + const auto past = CaptureStdout([&] { (void)flm::core::Factory(spec)(Context(p, 4096)); }); + TEST_REQUIRE(past.find("WARNING") != std::string::npos); + TEST_REQUIRE(past.find("4096 is past 2048: the answers drift") != std::string::npos); +} + void TestTheCatalogsGgufIsTheOneLoaded() { Runtime r; Package p({"a.gguf", "b.gguf"}); @@ -245,6 +274,7 @@ int main() { RUN(TestPreemptionIsRefusedBeforeTheWeightsAreOpened); RUN(TestAContextOutsideTheRuleIsRefusedBeforeTheWeightsAreOpened); RUN(TestACustomContextRuleIsApplied); + RUN(TestAContextPastTheReliableOneLoadsWithAWarning); RUN(TestTheCatalogsGgufIsTheOneLoaded); RUN(TestACatalogGgufThatIsMissingIsNamed); RUN(TestACatalogNamingTwoGgufsIsRefused); diff --git a/src/test/phi4_core/fake_corelib.cpp b/src/test/phi4_core/fake_corelib.cpp index be419f850..3f76d15f8 100644 --- a/src/test/phi4_core/fake_corelib.cpp +++ b/src/test/phi4_core/fake_corelib.cpp @@ -59,6 +59,11 @@ unsigned char FillFor(const void* source) { } ryzenai_corelib_status Status(std::string_view name) { + if (auto pending = state.fail_counts.find(std::string(name)); + pending != state.fail_counts.end() && pending->second > 0) { + --pending->second; + return ryzenai_corelib_status_failure; + } const auto configured = state.statuses.find(std::string(name)); return configured == state.statuses.end() ? state.default_status : configured->second; @@ -647,6 +652,7 @@ void Reset() { state.resolution_counts.clear(); state.call_counts.clear(); state.statuses.clear(); + state.fail_counts.clear(); state.lifetime_events.clear(); state.live_objects = 0; state.releases = 0; diff --git a/src/test/phi4_core/fake_corelib.hpp b/src/test/phi4_core/fake_corelib.hpp index e45f6a51f..e680e5eee 100644 --- a/src/test/phi4_core/fake_corelib.hpp +++ b/src/test/phi4_core/fake_corelib.hpp @@ -112,6 +112,9 @@ struct State { std::unordered_map resolution_counts; std::unordered_map call_counts; std::unordered_map statuses; + /// \brief how many more calls of a function fail before it answers as + /// `statuses` says + std::unordered_map fail_counts; std::vector lifetime_events; std::atomic live_objects{0}; std::atomic releases{0}; From f1e6b16cb83d055b469d7439cbf6e91483ea7614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 21:27:46 -0700 Subject: [PATCH 07/13] docs(core): every -core tag's file, context and known limits qwen.md lists every -core tag with the file it pulls by default, its public fallback, its context limit and its default, how a later turn continues, and the limits corelib and an AIE4 test machine found: Qwen3.6 past 32768, Qwen3.8 on long prompts, the paged decode drift, Gemma 4 past 65535, Llama on a very long repetitive prompt, and the retried prefill timeout. phi.md drops the 4096 ceiling: corelib runs Phi-4-mini to 131072 with LongRoPE's long factors past one page. Co-authored-by: Cursor --- docs/docs/models/phi.md | 9 +++------ docs/docs/models/qwen.md | 39 +++++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index 60e37026c..2906d4325 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -30,7 +30,7 @@ flm run phi4-mini-it:4b - **Backend:** `core` — the backend names the kernel provider; FastFlowLM reaches these kernels through AMD's `ryzenai_corelib` - **Source format:** GGUF, read directly by corelib's own `phi4mini` model, the way Qwen3, Qwen3.5, Qwen3.6 and Gemma 4 run. No ONNX model, no tensor manifest, and no converted or packed weight file is produced or shipped. - **Quantization:** whichever GGUF `flm pull` fetched. AMD's **`Q4_G128`** file, from Phi-4-mini's ONNX export, is packed exactly as it is, apart from its FP16 scales narrowing to BF16. The fallback, unsloth's **`Q8_0`**, is requantized to **group 64** on every load, which is a **lossy** second quantization step, so its output differs from the Q8_0 source. -- **Usable generation window:** 4095 tokens — the rendered prompt plus the requested output together, so the largest admissible prompt is 4094. An over-capacity request is rejected with HTTP 400 *before* any work is submitted to the device. Note this is far below the model's 128k context; see below for why. +- **Context:** up to 131072 tokens, and 131072 by default, the most corelib runs Phi-4-mini at. The usable generation window is one less than the context: the rendered prompt plus the requested output together. An over-capacity request is rejected with HTTP 400 *before* any work is submitted to the device. - **Availability:** Windows and Linux, and this is a **developer build**. The core runtime is not packaged by the MSI, Inno or snap installer; you build against corelib yourself. On core this tag pulls from pinned repositories, because the GGUF publishers do not ship the tokenizer files FastFlowLM's tokenizer frontend consumes. The GGUF comes from AMD's repository when `flm pull` can reach it: it is private for now, so that takes a Hugging Face token with access to it (`HF_TOKEN`, or `hf auth login`). Otherwise the pull falls back to unsloth's: @@ -84,14 +84,11 @@ flm pull phi4-mini-it-core:4b flm run phi4-mini-it-core:4b ``` -### Why the context is 4096, and why the usable window is one less +### Context, and why the usable window is one less -Phi-4-mini itself supports 128k, and the existing `phi4-mini-it:4b` tag defaults to 32k. This backend gives you 4095. That is a real functional regression and it has two separate causes, which are worth keeping apart. - -**The 4096 ceiling is a correctness boundary, not a buffer size.** 4096 is exactly Phi-4-mini's `rope.scaling.original_context_length`. LongRoPE selects its factors by *sequence length*, not per position: at or below the original length the short factors apply, above it the long ones do. corelib's `phi4mini` reads only the short factors, so 4096 is the point past which the rope tables would silently be wrong. It is enforced rather than assumed — corelib refuses a GGUF whose `phi3.rope.scaling.original_context_length` is not exactly 4096. Raising this ceiling means deriving the long factors, not enlarging an array. +corelib runs Phi-4-mini to its own 131072 tokens on paged attention, growing the KV cache a page at a time. LongRoPE selects its factors by *sequence length*, not per position: corelib ropes every position with the short factors at a capacity of one 4096-token page and with the long factors past it, as llama.cpp does, so a context set to 4096 or less takes the short ones and anything larger the long ones. A GGUF without `rope_factors_long` runs 4096 only. **The extra −1 is the core backend's.** It decodes at most one token fewer than its context, so that any request the server admits has room to finish rather than failing partway. It costs exactly one token. - ### No fallback Backend selection follows the tag, never a filename or a quantization level. The `-core` suffix *is* the request for corelib's kernels, and the entry's `supported_platforms` decides whether this machine is offered it at all. There is nothing to fall back to and nothing to resolve between: `phi4-mini-it:4b` and `phi4-mini-it-core:4b` are separate tags, so asking for one never gets you the other. Once this tag is selected, there is no fallback: if corelib is missing, unloadable, or the wrong version, the tag **fails to load with a diagnostic** rather than quietly running on CPU or on the NPU2/Q4NX backend. diff --git a/docs/docs/models/qwen.md b/docs/docs/models/qwen.md index f58b3555a..c996a473f 100644 --- a/docs/docs/models/qwen.md +++ b/docs/docs/models/qwen.md @@ -105,25 +105,36 @@ flm run qwen3:8b - **Tags:** `qwen3-core:0.6b`, `qwen3-core:1.7b`, `qwen3-core:4b`, `qwen3-core:8b` — their own tags, beside the NPU2 `qwen3:*` entries. `supported_platforms` is `dev04`, and the `-core` suffix is what asks for corelib's kernels. - **Backend:** `core` — one engine for all four sizes. The GGUF's own metadata selects the size and is cross-checked against it. -- **Source format:** GGUF `Q8_0`, read directly. Every weight is requantized to **group 64** while it is packed, except the 0.6B lm_head, which packs at **group 32**. -- **Usable generation window:** 4095 tokens, prompt plus output. Over-capacity requests are rejected with HTTP 400 before any work reaches the device. +- **Source format:** GGUF, read directly by corelib. With a Hugging Face token that can read AMD's private repositories (`HF_TOKEN`, or `hf auth login`), `flm pull` fetches AMD's 4-bit file, which packs exactly as it is; without one it falls back to the public GGUF. A public `Q8_0` is requantized to 4-bit groups on every load, which is lossy. +- **Context:** each tag's limit is the most corelib runs that model at; the default is lower where corelib or our own runs found answers going wrong past it. Over-capacity requests are rejected with HTTP 400 before any work reaches the device. - **Availability:** developer build against corelib 0.53.0 — see [Phi-4 on the core backend](phi.md). -The same corelib path also runs `gemma4-it-core:e2b` and `gemma4-it-core:e4b` (text only), `qwen3.5-core:9b` and `qwen3.6-moe-core:35b-a3b` (text only). Qwen3 stays at 4096; the other three go further. +The same corelib path runs every `-core` tag: -FastFlowLM sends a long prompt to these models in chunks of `max_prefill_len` (4096) tokens that end on multiples of 4096 counted from the start of the conversation, not from the start of the turn. A later turn's prompt therefore starts with at most one chunk inside a 4096-token page, and every chunk after that starts on a page boundary, which is where corelib runs a whole page as one pass. +| Tag | Default file | Public fallback | Context limit | Default context | +|---|---|---|---|---| +| `qwen3-core:0.6b` | unsloth `Q8_0`, refit to group 64 (lm_head 32) | — | 32768 | 32768 | +| `qwen3-core:1.7b` / `4b` / `8b` | AMD `Q4_G128` | unsloth `Q8_0` | 32768 | 32768 | +| `qwen3.5-core:4b` / `9b` | AMD `Q4_G128` | unsloth `Q8_0` | 262144 | 32768 | +| `qwen3.6-moe-core:35b-a3b` | AMD `Q4_G32` | unsloth `Q8_0` | 262144 | 32768 | +| `qwen3.8-core:27b` | AMD `Q4_G128`, on the NPU | unsloth `Q4_0`, on the GPU | 262144 | 32768 | +| `gemma4-it-core:e2b` / `e4b` | AMD `Q4_G32` | unsloth `Q8_0` | 131072 | 65536 | +| `phi4-mini-it-core:4b` | AMD `Q4_G128` | unsloth `Q8_0` | 131072 | 131072 | +| `llama3.2-core:1b` / `3b`, `llama3.1-core:8b` | `Q8_0`, refit to group 64 | — | 131072 | 131072 | -- **`qwen3.5-core:9b` — up to 32768 tokens.** The default is 4096; raise it with `--ctx-len` (`-c`). Past 4096 the context is rounded up to whole 4096-token pages. Long contexts have three costs: - - **Memory, up front.** The KV cache is allocated when the model is created, at the full capacity: 32 KiB per token (8 attention layers × 4 KV heads × 256 × BF16, for K and V), so 32768 tokens is 1 GiB on top of the weights. - - **Prompts run at decode speed.** corelib's Qwen3.5 has no batched prefill: every prompt token is one decode step, so a long prompt takes as long as generating that many tokens. - - **Accuracy past the first page.** Attention past 4096 tokens runs on a paged decode kernel that drifts a little further from the reference with each page it reads (corelib's `PAGED_DECODE_PAGE_DRIFT`). corelib reports Qwen3.5-4B and 9B still recalling a fact from position 0 at 31,983 tokens. -- **`qwen3.6-moe-core:35b-a3b` — up to 131072 tokens**, the most corelib has verified on the NPU (the model's own limit is 262144; corelib refuses more). The default is 4096; raise it with `--ctx-len` (`-c`). Long contexts have three costs: - - **Memory, up front.** The KV cache is allocated when the model is created, at the full capacity you ask for (rounded up to whole 4096-token pages), whether or not a conversation ever fills it: about 20 KiB per token (10 attention layers × 2 KV heads × 256 × BF16, for K and V), so 131072 tokens is about 2.5 GiB on top of the weights. - - **Long prompts run in passes of up to a page.** The first 4096 tokens from position 0 are one pass, and so is each later chunk that starts on a page boundary, padded to the page; past the second page these passes are slightly less accurate (corelib's `PAGED_PREFILL_DEEP_PAGES`, which corelib measured at perplexity 13.17 against 13.18 exact at 12,500 tokens). A chunk starting inside a page runs as passes up to the next page boundary, padded to at least 256 rows where that fits, also on the NPU; only a pass in a page's last 255 positions is padded to 128 rows and runs its attention on the CPU. corelib steps a few short remainders one token at a time instead: a pass of fewer than 5 tokens, a 128-row pass with only a few live tokens, and a pass starting in the last 127 positions of a page. - - **Checkpoints take host memory.** A checkpoint (what lets a conversation rewind without replaying) copies the KV cache into host memory, in whole 4096-token pages up to the current position: about 80 MiB per filled page, so up to about 2.5 GiB at a full 131072-token context. -- **`gemma4-it-core:e2b` / `gemma4-it-core:e4b` — up to 16384 tokens**, the length corelib has validated paged Gemma 4 to, and never past the GGUF's own `gemma4.context_length`. The default is 16384. Up to 4096 any length works; above 4096 the context must be a multiple of 4096. The KV cache for the whole context is allocated up front, when the model loads. +Every model's attention is paged: corelib keeps the KV cache in 4096-token pages and grows it a page at a time, so a large context costs memory only as a conversation fills it. Raise the context with `--ctx-len` (`-c`), up to the limit. -Qwen3, Qwen3.5, Qwen3.6, Gemma 4 and Phi-4 run as corelib's own models, which pack their weights on every load: there is no packed-weight cache beside the GGUF. A `.core-weights.bin` / `.json` pair left there by an earlier build is no longer read and can be deleted. +FastFlowLM sends a long prompt in chunks of `max_prefill_len` (4096) tokens that end on multiples of 4096 counted from the start of the conversation, not from the start of the turn, so every chunk after a turn's first starts on a page boundary, where corelib runs a whole page as one pass. Llama, Qwen3, Phi-4-mini and Gemma 4 run a later turn together with the earlier tokens of the page it starts in. Qwen3.5, Qwen3.6 and Qwen3.8 carry recurrent state and continue from where they are; going back to the last checkpoint restores a snapshot, which copies the filled KV pages into host memory, and going back anywhere else replays the conversation. + +Known limits, each corelib's (its `docs/known-problems.md` names them): + +- **Qwen3.6 past 32768.** Paged attention can return garbage: a passkey asked for after about 48k tokens came back as `!` repeated (on an AIE4 test machine). A context past 32768 still loads, and says so. +- **Qwen3.8 on long prompts.** A long prompt can come back as `!` repeated, run to run and from corelib's own runner too: on an AIE4 test machine a passkey was answered at 6k and 14k tokens and garbled at 10k and 28k. Prefill passes shorter than a page past position 0 run their attention on the CPU (`PAGED_PREFILL_SHORT_CHUNK`). +- **Qwen3.5 and Qwen3.8 decode** drift from the reference the more pages it reads (`PAGED_DECODE_PAGE_DRIFT`). +- **Gemma 4 past position 65535** (`PAGED_ATTENTION_POSITION_WRAP`). +- **Llama on a very long, repetitive prompt.** A passkey in about 96k tokens of one repeated sentence came back as `!` repeated, from corelib's own runner too; Phi-4-mini answered the same prompt. +- **A 4096-token prefill pass can fail to complete** (`PAGED_PREFILL_PREEMPTION_TIMEOUT`), more often while another process uses the NPU. corelib fails only that call, so FastFlowLM resets the session and runs the conversation again from the start, once; a second failure needs an unload and reload. +Every core model runs as corelib's own model, which packs its weights on every load: there is no packed-weight cache beside the GGUF. A `.core-weights.bin` / `.json` pair left there by an earlier build is no longer read and can be deleted. ```powershell flm pull qwen3-core:4b From 1fb2cd297e3f091462fb747ec62c128f0959a5c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 22:16:49 -0700 Subject: [PATCH 08/13] test(gemma4): stub gemma4e_npu_state_api in the frontend harness modeling_gemma4e.cpp attaches the prefix cache through gemma4e_npu_state_api since upstream's prefix slots for Gemma 4; only the engine exports it, so the harness returns a null table, which leaves the cache off, as the llama and phi4 harnesses do. Co-authored-by: Cursor --- src/test/gemma4_core/test_gemma4_frontend.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/gemma4_core/test_gemma4_frontend.cpp b/src/test/gemma4_core/test_gemma4_frontend.cpp index 98194911e..486c7ef6f 100644 --- a/src/test/gemma4_core/test_gemma4_frontend.cpp +++ b/src/test/gemma4_core/test_gemma4_frontend.cpp @@ -259,6 +259,9 @@ Sampler::Sampler(int features, sampler_config& config) void Sampler::reset_penalties() {} void Sampler::set_seed(uint64_t) {} void Sampler::set_repeat_last_n(size_t n) { repeat_last_n = n; } + +// No engine library is linked; a null table leaves the prefix cache off. +extern "C" const flm_state_api* gemma4e_npu_state_api(void*) { return nullptr; } int Sampler::sample(buffer&, const std::vector*, sample_logprobs_t*) { if (g_sample_index < g_samples.size()) return g_samples[g_sample_index++]; return 7; From 094cb14f8a2aefcc90607e3cedefd17cfdf57c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 27 Oct 2026 23:43:11 -0700 Subject: [PATCH 09/13] chore(core): pin Qwen3.8 and Qwen3.6's AMD Q4 GGUFs to their published commits amd/Qwen3.8-27B-Q4_G128-GGUF at deab6eaf and amd/Qwen3.6-35B-A3B-Q4_G32-GGUF at 53a2155b, in place of main: every file there matches model_info.json's size and sha256. Co-authored-by: Cursor --- src/model_list.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/model_list.json b/src/model_list.json index 529a5e79f..30f94ded5 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -1940,8 +1940,8 @@ } }, "preferred_source": { - "url": "https://huggingface.co/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/resolve/main", - "file_url": "https://huggingface.co/api/models/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/tree/main", + "url": "https://huggingface.co/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/resolve/53a2155ba4237f4bffc3914b2f63cbb0bc68585b", + "file_url": "https://huggingface.co/api/models/amd/Qwen3.6-35B-A3B-Q4_G32-GGUF/tree/53a2155ba4237f4bffc3914b2f63cbb0bc68585b", "use_hf_token": true, "size": 21311733654, "footprint": 21.3, @@ -2010,8 +2010,8 @@ } }, "preferred_source": { - "url": "https://huggingface.co/amd/Qwen3.8-27B-Q4_G128-GGUF/resolve/main", - "file_url": "https://huggingface.co/api/models/amd/Qwen3.8-27B-Q4_G128-GGUF/tree/main", + "url": "https://huggingface.co/amd/Qwen3.8-27B-Q4_G128-GGUF/resolve/deab6eafc90811fc6738db7c05ec615130074cb3", + "file_url": "https://huggingface.co/api/models/amd/Qwen3.8-27B-Q4_G128-GGUF/tree/deab6eafc90811fc6738db7c05ec615130074cb3", "use_hf_token": true, "size": 15977762336, "footprint": 15.96, From e7621f7175dd18b9fb95036eb3b562f0abd331ee Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Wed, 28 Oct 2026 15:31:56 -0700 Subject: [PATCH 10/13] feat(platform): ask corelib which NPU generation this machine has flm list offered whatever default_npu_platform() was compiled to: dev04 in every corelib build, aie2p in every other, whatever the silicon. A corelib build now asks corelib, once per process, which NPU it has: AIEM is dev04, and AIE2P, none, or a corelib that cannot be loaded is aie2p. corelib reads the device itself, so FastFlowLM keeps no table of devices, IDs or geometry; a build without corelib answers aie2p, as every dev04 model is a corelib one. The question goes through ryzenai_corelib_testing_npu_arch, the one entry point that names the architecture whatever the NPU stack's version, until corelib's public header has one; the header is included after the version pin, which a testing hook makes the more necessary. CorelibRuntime::NpuArch loads the API without the readiness check, which refuses an AIE2P, and GetOrCreate reuses that load. On aie2p, main() no longer reports corelib as unavailable for every run: there it has nothing to run. On this AIE2P machine a corelib build lists the FastFlowLM models and none of the -core ones. Co-authored-by: Cursor --- docs/docs/models/phi.md | 2 +- src/common/core/corelib_runtime.cpp | 38 ++++++++++++++----- src/common/models/README.md | 28 ++++++-------- src/common/npu_platform.cpp | 17 ++++++--- src/include/core/corelib_api.hpp | 5 +++ src/include/core/corelib_runtime.hpp | 6 +++ src/include/utils/npu_platform.hpp | 36 ++++++------------ src/src/main.cpp | 6 ++- .../test_model_list_platform.cpp | 13 ++++--- src/test/phi4_core/fake_corelib.cpp | 3 ++ src/test/phi4_core/fake_corelib.hpp | 2 + src/test/phi4_core/test_corelib_api.cpp | 18 +++++++++ 12 files changed, 109 insertions(+), 65 deletions(-) diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index 2906d4325..0cecc3e94 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -26,7 +26,7 @@ flm run phi4-mini-it:4b ## 🧪 Model Card: Phi-4-mini-instruct on the core backend (developer preview) -- **Tag:** `phi4-mini-it-core:4b` — its own tag, beside `phi4-mini-it:4b`, which stays the NPU2/Q4NX build. They are two entries in the one catalog, `model_list.json`, describing two different packages, and nothing about one is a patch on the other. Two things decide whether an install offers this one, and **the first is the machine**: `supported_platforms` says `dev04` here and `aie2p` there, checked against the generation FastFlowLM reads from the device, so on shipping silicon this tag is not listed even by a build that has corelib compiled in. Only then does the second apply — whether corelib's kernels were linked at all, which the `-core` suffix on the family name is what asks for. `utils::get_device()` is still a stand-in and currently answers `dev04` for every build, so a corelib developer build offers this tag and not the NPU2 one; a build without corelib on the same setting offers no models at all, and says so. +- **Tag:** `phi4-mini-it-core:4b` — its own tag, beside `phi4-mini-it:4b`, which stays the NPU2/Q4NX build. They are two entries in the one catalog, `model_list.json`, describing two different packages, and nothing about one is a patch on the other. Two things decide whether an install offers this one, and **the first is the machine**: `supported_platforms` says `dev04` here and `aie2p` there, checked against the generation FastFlowLM reads from the device, so on shipping silicon this tag is not listed even by a build that has corelib compiled in. Only then does the second apply — whether corelib's kernels were linked at all, which the `-core` suffix on the family name is what asks for. A corelib build asks corelib which NPU the machine has, so on a dev04 NPU it offers this tag and not the NPU2 one, and on an aie2p NPU the reverse. - **Backend:** `core` — the backend names the kernel provider; FastFlowLM reaches these kernels through AMD's `ryzenai_corelib` - **Source format:** GGUF, read directly by corelib's own `phi4mini` model, the way Qwen3, Qwen3.5, Qwen3.6 and Gemma 4 run. No ONNX model, no tensor manifest, and no converted or packed weight file is produced or shipped. - **Quantization:** whichever GGUF `flm pull` fetched. AMD's **`Q4_G128`** file, from Phi-4-mini's ONNX export, is packed exactly as it is, apart from its FP16 scales narrowing to BF16. The fallback, unsloth's **`Q8_0`**, is requantized to **group 64** on every load, which is a **lossy** second quantization step, so its output differs from the Q8_0 source. diff --git a/src/common/core/corelib_runtime.cpp b/src/common/core/corelib_runtime.cpp index 30fd8f103..ccf32c220 100644 --- a/src/common/core/corelib_runtime.cpp +++ b/src/common/core/corelib_runtime.cpp @@ -7,6 +7,22 @@ namespace flm::corelib { namespace { std::mutex process_mutex; std::shared_ptr process_runtime; +/// \brief the API loaded for this process, kept so a runtime that failed to +/// come up, or the NPU probe, does not load it again +std::shared_ptr process_api; + +/// \note The caller holds process_mutex. +std::shared_ptr ProcessApi(const std::filesystem::path& executable_dir) { + if (!process_api) { +#if defined(FLM_CORELIB_LINK_STATIC) + (void)executable_dir; + process_api = CorelibApi::LoadStatic(); +#else + process_api = CorelibApi::Load(CorelibApi::ResolveLibraryPath(executable_dir)); +#endif + } + return process_api; +} #if defined(FLM_CORELIB_TESTING) std::function destruction_observer; bool shutdown_execution_lock_held = false; @@ -38,18 +54,21 @@ std::shared_ptr CorelibRuntime::CreateReady( std::shared_ptr CorelibRuntime::GetOrCreate( const std::filesystem::path& executable_dir) { std::lock_guard lock(process_mutex); - if (!process_runtime) { -#if defined(FLM_CORELIB_LINK_STATIC) - (void)executable_dir; - auto api = CorelibApi::LoadStatic(); -#else - auto api = CorelibApi::Load(CorelibApi::ResolveLibraryPath(executable_dir)); -#endif - process_runtime = CreateReady(std::move(api)); - } + if (!process_runtime) process_runtime = CreateReady(ProcessApi(executable_dir)); return process_runtime; } +std::string CorelibRuntime::NpuArch(const std::filesystem::path& executable_dir) noexcept { + try { + std::lock_guard lock(process_mutex); + const auto api = process_runtime ? process_runtime->api_ : ProcessApi(executable_dir); + const char* arch = api->functions().npu_arch(); + return arch ? arch : ""; + } catch (...) { + return ""; + } +} + std::shared_ptr CorelibRuntime::CreateForTest( std::shared_ptr api) { auto runtime = CreateReady(std::move(api)); @@ -63,6 +82,7 @@ std::shared_ptr CorelibRuntime::CreateForTest( void CorelibRuntime::ShutdownProcess() { std::lock_guard process_lock(process_mutex); + process_api.reset(); if (!process_runtime) return; auto runtime = process_runtime; diff --git a/src/common/models/README.md b/src/common/models/README.md index 17cff6aab..530fa58d9 100644 --- a/src/common/models/README.md +++ b/src/common/models/README.md @@ -23,16 +23,14 @@ flow it was configured with, and several flows can serve one generation. Keying either axis off the other is how adding corelib once took the FastFlowLM models away. -`get_device()` is a **stand-in** until the real probe lands, and it consults -nothing: it returns `default_npu_platform()`. Not the environment, and *not* -whether corelib was linked — linking corelib says what kernels this binary has, -never what silicon it is running on, and an install that lists a model it -cannot run is worse than one that lists nothing. So the constant *is* the -answer, for every build: change it and rebuild to move a whole install to the -other generation, and the probe replaces the one function. - -It is **`dev04`** today, while the corelib path is being brought up. Read -the next paragraph before being surprised by what `flm list` shows. +`get_device()` asks the machine, once per process. A build that links corelib +asks corelib what NPU it has (`CorelibRuntime::NpuArch`, corelib's +`ryzenai_corelib_testing_npu_arch` until its public header names one): `AIEM` +is `dev04`, and `AIE2P`, no NPU, or a corelib that cannot be loaded is +`default_npu_platform()`, `aie2p`. corelib reads the device itself, so +FastFlowLM holds no table of devices and names none. A build without corelib +answers `aie2p`: every `dev04` model is a corelib one. Linking corelib still +says nothing about the silicon — it is what lets the build ask. This is a contributor document. For *using* a backend once it exists — `--backend`, `FLM_BACKEND`, precedence — see [`docs/docs/instructions/cli.md`](../../../docs/docs/instructions/cli.md). @@ -379,13 +377,9 @@ Points that are easy to get wrong: outside a pull, the entry resolves to the preferred source exactly when every file only it has is on disk. - The two keys are checked independently, and **the platform is checked first - and for everyone**. On today's `dev04` default that means a corelib build - offers the `-core` tags and nothing else, and a build *without* corelib - offers **nothing at all** — the FastFlowLM entries are pruned by the - generation, and the `-core` entries by the kernels they would need. That is the - cost of bringing up the next generation before its probe exists, and it is - reversed by setting `default_npu_platform()` back to `aie2p`, which gives - both builds the same FastFlowLM tags and none of the `-core` ones. + and for everyone**. On a `dev04` NPU a corelib build offers the `-core` tags + and nothing else; on an `aie2p` one, or none, every build offers the + FastFlowLM tags and none of the `-core` ones. - An empty catalog is therefore an ordinary state, not a crash: `model_list` prints one line naming the generation and the linked kernels, `flm list` says it found nothing, `flm run` says the tag is not found, and `flm --help` still diff --git a/src/common/npu_platform.cpp b/src/common/npu_platform.cpp index c86ccc984..c2d0b8bd6 100644 --- a/src/common/npu_platform.cpp +++ b/src/common/npu_platform.cpp @@ -2,16 +2,21 @@ /// \brief which NPU generation this host has (aie2p vs dev04) #include "utils/npu_platform.hpp" +#if defined(FLM_ENABLE_CORE) +#include "core/corelib_runtime.hpp" +#include "utils/utils.hpp" +#endif + namespace utils { npu_platform get_device() { - // No device read here: the probe that tells the generations apart is not in - // this tree, and nothing else in the build is allowed to stand in for it. - // Linking corelib in particular is not evidence -- which kernels were - // compiled is a fact about the build, the generation is a fact about the - // machine, and letting the first answer the second is how the two axes get - // quietly welded back together. +#if defined(FLM_ENABLE_CORE) + static const npu_platform platform = platform_from_corelib_arch( + flm::corelib::CorelibRuntime::NpuArch(get_executable_directory())); + return platform; +#else return default_npu_platform(); +#endif } std::optional parse_platform(std::string_view text) { diff --git a/src/include/core/corelib_api.hpp b/src/include/core/corelib_api.hpp index 45e23dc06..404f5bc40 100644 --- a/src/include/core/corelib_api.hpp +++ b/src/include/core/corelib_api.hpp @@ -7,6 +7,10 @@ #error "FastFlowLM requires ryzenai-corelib headers exactly 0.53.0" #endif +// ryzenai_corelib_testing_npu_arch, until corelib's public header names the +// NPU's architecture. After the pin: a testing hook may change in any release. +#include + #include #include #include @@ -23,6 +27,7 @@ X(get_last_error_message, ryzenai_corelib_get_last_error_message) \ X(selftest_dependencies, ryzenai_corelib_selftest_dependencies) \ X(npu_usable, ryzenai_corelib_npu_usable) \ + X(npu_arch, ryzenai_corelib_testing_npu_arch) \ X(object_release, ryzenai_corelib_object_release) \ X(create_stream, ryzenai_corelib_create_stream) \ X(stream_synchronize, ryzenai_corelib_stream_synchronize) \ diff --git a/src/include/core/corelib_runtime.hpp b/src/include/core/corelib_runtime.hpp index b4e5e0418..6e08299bf 100644 --- a/src/include/core/corelib_runtime.hpp +++ b/src/include/core/corelib_runtime.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #if defined(FLM_CORELIB_TESTING) #include #endif @@ -18,6 +19,11 @@ class CorelibRuntime final { const std::filesystem::path& executable_dir); static std::shared_ptr CreateForTest( std::shared_ptr api); + /// \brief this machine's NPU as corelib names it: "AIEM", "AIE2P", or "" + /// when it knows none or cannot be loaded + /// \note Needs no usable NPU and opens no device, so it answers where + /// GetOrCreate would refuse; it loads the API GetOrCreate then reuses. + static std::string NpuArch(const std::filesystem::path& executable_dir) noexcept; static void ShutdownProcess(); #if defined(FLM_CORELIB_TESTING) static void SetDestructionObserverForTest(std::function observer); diff --git a/src/include/utils/npu_platform.hpp b/src/include/utils/npu_platform.hpp index f5f8a0d14..96aab6560 100644 --- a/src/include/utils/npu_platform.hpp +++ b/src/include/utils/npu_platform.hpp @@ -32,35 +32,21 @@ constexpr std::string_view platform_id(npu_platform platform) { : std::string_view("aie2p"); } -/// \brief the generation assumed when nothing has read the device -/// \note Until the probe in get_device() is real this is not a default but the -/// whole answer, so it decides what every install can see. It is -/// dev04 while the corelib path is what is being brought up: that is -/// the generation corelib's kernels are built for, and an install that -/// claimed aie2p could not offer them at all. -/// \note The cost is exact and intended: every entry in model_list.json except -/// the corelib one names aie2p, so a build that does not link corelib -/// offers nothing here. That is not a broken install, it is an install -/// for silicon none of its models were built for, and model_list says so -/// in one line. Set this back to aie2p to get those 42 models back. -constexpr npu_platform default_npu_platform() { -#if defined(FLM_ENABLE_CORE) - return npu_platform::dev04; -#else - return npu_platform::aie2p; -#endif +/// \brief the generation assumed when nothing identifies one +constexpr npu_platform default_npu_platform() { return npu_platform::aie2p; } + +/// \brief the generation corelib's name for this machine's NPU stands for +/// \param arch what corelib reports: "AIEM", "AIE2P", or "" for none it knows +constexpr npu_platform platform_from_corelib_arch(std::string_view arch) { + return arch == "AIEM" ? npu_platform::dev04 : default_npu_platform(); } /// \brief ask the machine which NPU generation it has /// \return the generation of the NPU in this host -/// \note A stand-in, and deliberately the dumbest one that can be written: the -/// real probe reads the device and is not part of this tree, so until it -/// lands this returns default_npu_platform() and consults nothing at all -/// -- not the environment, and not which kernels were linked. A build -/// that links corelib is still a build on whatever silicon it is running -/// on, and an install that offers a model it cannot run is worse than one -/// that offers nothing. The call sites need no change when the probe -/// arrives: they already treat the generation as a run-time answer. +/// \note A build that links corelib asks corelib, which reads the device +/// itself: FastFlowLM keeps no table of devices. Every dev04 model is a +/// corelib one, so a build without corelib, or one whose corelib cannot +/// be loaded, answers default_npu_platform(). Asked once per process. npu_platform get_device(); /// \brief parse a catalog platform id diff --git a/src/src/main.cpp b/src/src/main.cpp index 7ae61a332..4912ccff2 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -665,8 +665,10 @@ int main(int argc, char* argv[]) { // Say once, here, why there is no device. The commands below cannot run // without one, and the message they eventually produce names the symptom // rather than the cause: it is raised from whichever model first reaches - // for the device, long after the runtime that failed to come up. - if (print_status && needs_npu && !corelib_note.empty()) { + // for the device, long after the runtime that failed to come up. Only on + // dev04: elsewhere corelib has nothing to run and its absence is expected. + if (print_status && needs_npu && !corelib_note.empty() && + platform == utils::npu_platform::dev04) { header_print("FLM", "core backend unavailable (" << corelib_note << ")"); } // Only print early error for RAI builds where the device is opened eagerly. diff --git a/src/test/model_list_platform/test_model_list_platform.cpp b/src/test/model_list_platform/test_model_list_platform.cpp index 9e21a59a6..ffd328518 100644 --- a/src/test/model_list_platform/test_model_list_platform.cpp +++ b/src/test/model_list_platform/test_model_list_platform.cpp @@ -240,12 +240,15 @@ void test_platform_helpers() { TEST_REQUIRE(!utils::parse_platform("stx").has_value()); TEST_REQUIRE(utils::parse_platform(utils::platform_id(utils::npu_platform::dev04)) == utils::npu_platform::dev04); - TEST_REQUIRE(utils::default_npu_platform() == utils::npu_platform::dev04); + TEST_REQUIRE(utils::default_npu_platform() == utils::npu_platform::aie2p); - // The stand-in consults nothing -- not the environment, not which kernels - // were linked -- so it is the default here and in a corelib build alike. - // Changing what an install can see means changing default_npu_platform() - // and rebuilding, which is the one place to look. + // corelib names the NPU; only its AIEM is dev04, and anything else, + // no NPU included, is the default. + TEST_REQUIRE(utils::platform_from_corelib_arch("AIEM") == utils::npu_platform::dev04); + TEST_REQUIRE(utils::platform_from_corelib_arch("AIE2P") == utils::npu_platform::aie2p); + TEST_REQUIRE(utils::platform_from_corelib_arch("") == utils::npu_platform::aie2p); + + // Built without corelib, as this test is, nothing can name a dev04. TEST_REQUIRE(utils::get_device() == utils::default_npu_platform()); TEST_REQUIRE(utils::get_device() == utils::get_device()); } diff --git a/src/test/phi4_core/fake_corelib.cpp b/src/test/phi4_core/fake_corelib.cpp index 3f76d15f8..f85a85bfd 100644 --- a/src/test/phi4_core/fake_corelib.cpp +++ b/src/test/phi4_core/fake_corelib.cpp @@ -176,6 +176,8 @@ struct TypedFake { return state.selftest_status; } else if constexpr (std::is_same_v) { return state.npu_usable; + } else if constexpr (std::is_same_v) { + return state.npu_arch.c_str(); } else if constexpr (std::is_same_v) { void* object = std::get<0>(arguments); if (object) { @@ -643,6 +645,7 @@ void Reset() { state.selftest_status = ryzenai_corelib_status_success; state.default_status = ryzenai_corelib_status_success; state.npu_usable = true; + state.npu_arch = "AIEM"; state.stream_prefill_pdis.clear(); state.stream_token_pdis.clear(); state.detail.clear(); diff --git a/src/test/phi4_core/fake_corelib.hpp b/src/test/phi4_core/fake_corelib.hpp index e680e5eee..4fd01e589 100644 --- a/src/test/phi4_core/fake_corelib.hpp +++ b/src/test/phi4_core/fake_corelib.hpp @@ -100,6 +100,8 @@ struct State { ryzenai_corelib_status default_status{ryzenai_corelib_status_success}; /// \brief what npu_usable() answers bool npu_usable{true}; + /// \brief what testing_npu_arch() answers + std::string npu_arch{"AIEM"}; /// \brief the PDI lists the engine opened its stream with /// \note Required from 0.5.0 and deliberately undefaulted by corelib, so /// it is worth being able to assert on. diff --git a/src/test/phi4_core/test_corelib_api.cpp b/src/test/phi4_core/test_corelib_api.cpp index 431361405..46dda6710 100644 --- a/src/test/phi4_core/test_corelib_api.cpp +++ b/src/test/phi4_core/test_corelib_api.cpp @@ -235,6 +235,23 @@ void TestRuntimeRunsDependencySelftestAndRequiresAUsableNpu() { CorelibRuntime::ShutdownProcess(); } +void TestNpuArchIsCorelibsAnswerAndEmptyWhenItCannotLoad() { + fake_corelib::Reset(); + { + const auto runtime = CorelibRuntime::CreateForTest(ValidApi()); + fake_corelib::GetState().npu_arch = "AIE2P"; + TEST_REQUIRE(CorelibRuntime::NpuArch({}) == "AIE2P"); + fake_corelib::GetState().npu_arch = "AIEM"; + TEST_REQUIRE(CorelibRuntime::NpuArch({}) == "AIEM"); + } + CorelibRuntime::ShutdownProcess(); + + const auto empty = std::filesystem::temp_directory_path() / "flm-no-corelib-here"; + std::filesystem::create_directories(empty); + TEST_REQUIRE(CorelibRuntime::NpuArch(empty).empty()); + std::filesystem::remove_all(empty); +} + void TestExecutionLeaseSerializesTwoThreads() { fake_corelib::Reset(); const auto runtime = CorelibRuntime::CreateForTest(ValidApi()); @@ -315,6 +332,7 @@ int main() { RUN_TEST(TestFallbackIsExeDirectoryCoreDllNotCurrentDirectory); RUN_TEST(TestEveryUniqueObjectReleasesExactlyOnceAfterMoves); RUN_TEST(TestRuntimeRunsDependencySelftestAndRequiresAUsableNpu); + RUN_TEST(TestNpuArchIsCorelibsAnswerAndEmptyWhenItCannotLoad); RUN_TEST(TestExecutionLeaseSerializesTwoThreads); RUN_TEST(TestShutdownReleasesExecutionLockBeforeDestroyingRuntimeOwner); RUN_TEST(TestCleanupRunsAfterTheLastObjectAndOnlyOnce); From f82840f60d5d9e813734a549726f8fcd9a080d2a Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Wed, 28 Oct 2026 17:41:24 -0700 Subject: [PATCH 11/13] test(phi4): count the prefill's sample toward the budget, as main's _shared_generate now does #870 counts the token insert() sampled toward generated_tokens and the length limit. A 4093-token prompt in a 4095-token window now reports 3 generated tokens, the engine still running 11 and 12; and a budget of 2 is what leaves 12 sampled but never run, as a budget of 1 did. Co-authored-by: Cursor --- src/test/phi4_core/test_phi4_frontend.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/phi4_core/test_phi4_frontend.cpp b/src/test/phi4_core/test_phi4_frontend.cpp index 6c3807103..8f3ad83a6 100644 --- a/src/test/phi4_core/test_phi4_frontend.cpp +++ b/src/test/phi4_core/test_phi4_frontend.cpp @@ -650,7 +650,8 @@ void TestOmittedZeroAndNegativeSentinelBudgetsCapAtRemainingWindow() { auto meta = Meta(); auto input = Input(requested); std::ostringstream output; TEST_REQUIRE(model->insert(meta, input)); (void)model->generate(meta, 4096, output); - TEST_REQUIRE(meta.generated_tokens == 2); + // 11, sampled from the prefill, counts; 12 and 13 fill the window. + TEST_REQUIRE(meta.generated_tokens == 3); TEST_REQUIRE(meta.stop_reason == MAX_LENGTH_REACHED); } @@ -660,7 +661,7 @@ void TestOmittedZeroAndNegativeSentinelBudgetsCapAtRemainingWindow() { g_encoded_tokens.assign(4093, 1); g_samples = {11, 12, 13}; g_sample_index = 0; auto meta = Meta(); auto input = Input(); std::ostringstream output; (void)model->generate_with_prompt(meta, input, 4096, output); - TEST_REQUIRE(meta.generated_tokens == 2); + TEST_REQUIRE(meta.generated_tokens == 3); } void TestCancellationBeforePrefillSubmitsNothing() { @@ -830,7 +831,8 @@ void TestATokenCutOffBeforeItRanIsPrefilledWithTheNextTurn() { (void)model->generate(meta, 10, output, [&] { return g_factory.engine->forward_calls >= 1; }); } else { - (void)model->generate(meta, 1, output); + // 11, sampled from the prefill, counts toward the budget. + (void)model->generate(meta, 2, output); } TEST_REQUIRE(meta.stop_reason == (cancel ? CANCEL_DETECTED : MAX_LENGTH_REACHED)); TEST_REQUIRE((g_factory.engine->forwarded == std::vector{11})); From 71d2c0ca793de5ec75f2e420ac6ec01b373e1602 Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Wed, 28 Oct 2026 17:51:51 -0700 Subject: [PATCH 12/13] fix(validate): check the NPU stack on aie2p in a corelib build too flm validate in a corelib build skipped every check and answered checks_skipped / ready, on any NPU: the skip was decided at compile time, when every corelib build was taken for dev04. Lemonade gates FastFlowLM's install on validate, so on XDNA2 it no longer checked the driver, the firmware or memlock. The skip is now the machine's: only a dev04 NPU, where corelib owns device setup, skips (and says npu_platform), and an aie2p one runs the same checks as a build without corelib. validate also runs before the device is opened, as it does in that build, since on Linux it probes /dev/accel itself and amdxdna refuses a second open. On an XDNA2 machine with driver 32.0.203.281, below the 32.0.203.304 minimum, a corelib build now reports npu_driver_ok false and ready false where it reported the checks skipped. Co-authored-by: Cursor --- src/src/main.cpp | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/src/main.cpp b/src/src/main.cpp index 4912ccff2..734864932 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -222,35 +222,38 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { #ifdef FLM_ENABLE_CORE // Everything below reads the NPU's geometry and this process's memlock // limit from the driver and holds them against what FastFlowLM's own kernel - // flow needs. A corelib build does not use that flow. corelib owns device + // flow needs. On dev04 only corelib's models run, and corelib owns device // setup and brings the NPU up itself -- main() already reports whether that // worked, and says why when it did not -- so these checks answer a question - // this build does not ask, against a threshold that describes FastFlowLM's - // own kernels rather than the parts corelib supports. Left in, they fail a - // working core install on hardware corelib runs on perfectly well. + // that NPU does not ask, against a threshold that describes FastFlowLM's + // own kernels rather than the parts corelib supports. On aie2p a corelib + // build runs FastFlowLM's own models, and they are checked as in any other. // // The JSON keeps the shape callers already parse, but reports the checks as - // skipped rather than claiming they passed: "ready" here means "this build + // skipped rather than claiming they passed: "ready" here means "this NPU // does not gate on the stock stack". A caller that wants to know whether the // NPU actually came up should look at whether flm opened a device. - if (json_output) { - nlohmann::json validation_json = { - {"object", "npu_stack_validation"}, + if (utils::get_device() == utils::npu_platform::dev04) { + if (json_output) { + nlohmann::json validation_json = { + {"object", "npu_stack_validation"}, #ifdef _WIN32 - {"platform", "windows"}, + {"platform", "windows"}, #else - {"platform", "linux"}, + {"platform", "linux"}, #endif - {"backend", "core"}, - {"checks_skipped", true}, - {"ready", true} - }; - std::cout << validation_json.dump(4) << std::endl; - } else if (print_human) { - header_print("FLM", "corelib backend: skipping NPU stack checks (corelib owns device setup)"); + {"npu_platform", std::string(utils::platform_id(utils::npu_platform::dev04))}, + {"backend", "core"}, + {"checks_skipped", true}, + {"ready", true} + }; + std::cout << validation_json.dump(4) << std::endl; + } else if (print_human) { + header_print("FLM", "corelib backend: skipping NPU stack checks (corelib owns device setup)"); + } + return true; } - return true; -#else +#endif #ifndef _WIN32 nlohmann::json validation_json = { {"object", "npu_stack_validation"}, @@ -497,7 +500,6 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { } return true; #endif -#endif // FLM_ENABLE_CORE } @@ -604,6 +606,11 @@ int main(int argc, char* argv[]) { // Declared before anything that can return so the destructor covers the // early exits and the catch-all below. CoreProcessGuard core_guard; + // Before this process takes the device, as in the build without corelib: + // validate probes /dev/accel/* itself, and amdxdna refuses a second open. + if (parsed_args.command == "validate") { + return sanity_check_npu_stack(false, parsed_args.json_output) ? 0 : 1; + } // Declared out here so the early exits below still see a device, and so a // corelib that fails to come up leaves it null rather than undefined. flm_rt::device* npu_device = nullptr; From 3a2be0f32740a2cae7fc12db5a74cda84536972f Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Thu, 29 Oct 2026 03:56:31 -0700 Subject: [PATCH 13/13] fix(validate): ask corelib whether a dev04 NPU can run, rather than skip On dev04, flm validate skipped every check and answered ready, so a caller gating on it, Lemonade among them, learned nothing about the NPU stack. The stock checks do not apply there -- they hold an XDNA2 device ID, driver line, firmware and column count against FastFlowLM's own kernels -- but corelib's own gate does: validate now brings corelib up as the core backend does, which loads XRT and asks whether this is an NPU whose stack corelib supports (on Windows, XRT and the NPU driver at corelib's minimums), opening no context. ready is that answer, and on Linux the memlock limit too, shared with the aie2p checks. The JSON reports corelib_npu_ok, and corelib_error when it is false, in place of checks_skipped. Co-authored-by: Cursor --- src/src/main.cpp | 137 ++++++++++++++++++++++++++--------------------- 1 file changed, 77 insertions(+), 60 deletions(-) diff --git a/src/src/main.cpp b/src/src/main.cpp index 734864932..d4a057b98 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -217,41 +217,90 @@ std::string identify_npu_arch() { #endif +#ifndef _WIN32 +///@brief checks this process's memlock limit, raising it to infinity when it can +///@return whether the limit is at least 100 MB once raised +static bool check_memlock(nlohmann::json& validation_json, bool print_human) { + bool memlock_ok = true; + struct rlimit rl; + if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) { + validation_json["memlock_limit"] = (rl.rlim_cur == RLIM_INFINITY) + ? "infinity" + : std::to_string(static_cast(rl.rlim_cur)); + if (rl.rlim_cur != RLIM_INFINITY && rl.rlim_cur < 100 * 1024 * 1024) { + struct rlimit rl_new = { RLIM_INFINITY, RLIM_INFINITY }; + if (setrlimit(RLIMIT_MEMLOCK, &rl_new) == 0) { + rl.rlim_cur = RLIM_INFINITY; + rl.rlim_max = RLIM_INFINITY; + validation_json["memlock_limit"] = "infinity"; + if (print_human) + header_print_g("Linux", "Memlock Limit: set to infinity"); + } else { + if (print_human) { + header_print_r("ERROR", "Memlock limit is too low (" << (rl.rlim_cur / 1024 / 1024) << "MB). Please raise the limit or set to infinity."); + } + memlock_ok = false; + } + } else if (print_human) { + if (rl.rlim_cur == RLIM_INFINITY) { + header_print_g("Linux", "Memlock Limit: infinity"); + } else { + header_print("Linux", "Memlock Limit: " << (rl.rlim_cur / 1024 / 1024) << " MB"); + } + } + } else { + if (print_human) + perror("Failed to get memlock limit"); + } + validation_json["memlock_ok"] = memlock_ok; + return memlock_ok; +} +#endif + static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { bool print_human = !quiet && !json_output; #ifdef FLM_ENABLE_CORE - // Everything below reads the NPU's geometry and this process's memlock - // limit from the driver and holds them against what FastFlowLM's own kernel - // flow needs. On dev04 only corelib's models run, and corelib owns device - // setup and brings the NPU up itself -- main() already reports whether that - // worked, and says why when it did not -- so these checks answer a question - // that NPU does not ask, against a threshold that describes FastFlowLM's - // own kernels rather than the parts corelib supports. On aie2p a corelib - // build runs FastFlowLM's own models, and they are checked as in any other. - // - // The JSON keeps the shape callers already parse, but reports the checks as - // skipped rather than claiming they passed: "ready" here means "this NPU - // does not gate on the stock stack". A caller that wants to know whether the - // NPU actually came up should look at whether flm opened a device. + // The stock checks below hold an XDNA2 device ID, driver line, firmware and + // column count against what FastFlowLM's own kernels need. On dev04 only + // corelib's models run, so the NPU is ready when corelib says it can run + // them: the same gate the core backend comes up behind, which loads XRT + // and asks whether the NPU stack is one corelib supports, opening no + // context. On aie2p a corelib build runs FastFlowLM's own models, and they + // are checked as in any other. if (utils::get_device() == utils::npu_platform::dev04) { - if (json_output) { - nlohmann::json validation_json = { - {"object", "npu_stack_validation"}, + nlohmann::json validation_json = { + {"object", "npu_stack_validation"}, #ifdef _WIN32 - {"platform", "windows"}, + {"platform", "windows"}, #else - {"platform", "linux"}, + {"platform", "linux"}, #endif - {"npu_platform", std::string(utils::platform_id(utils::npu_platform::dev04))}, - {"backend", "core"}, - {"checks_skipped", true}, - {"ready", true} - }; + {"npu_platform", std::string(utils::platform_id(utils::npu_platform::dev04))}, + {"backend", "core"} + }; + bool corelib_npu_ok = true; + try { + (void)flm::corelib::CorelibRuntime::GetOrCreate( + std::filesystem::path(utils::get_executable_directory())); + } catch (const std::exception& e) { + corelib_npu_ok = false; + validation_json["corelib_error"] = e.what(); + if (print_human) + header_print_r("ERROR", "corelib cannot run on this NPU: " << e.what()); + } + validation_json["corelib_npu_ok"] = corelib_npu_ok; + if (corelib_npu_ok && print_human) + header_print_g("FLM", "corelib: NPU usable"); + bool ready = corelib_npu_ok; +#ifndef _WIN32 + ready = check_memlock(validation_json, print_human) && ready; +#endif + if (print_human) + header_print_g("FLM", "NPU platform: " << validation_json["npu_platform"].get()); + validation_json["ready"] = ready; + if (json_output) std::cout << validation_json.dump(4) << std::endl; - } else if (print_human) { - header_print("FLM", "corelib backend: skipping NPU stack checks (corelib owns device setup)"); - } - return true; + return ready; } #endif #ifndef _WIN32 @@ -403,39 +452,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { header_print_r("ERROR", "No NPU device found."); } - // Check memlock limit - bool memlock_ok = true; - struct rlimit rl; - if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) { - validation_json["memlock_limit"] = (rl.rlim_cur == RLIM_INFINITY) - ? "infinity" - : std::to_string(static_cast(rl.rlim_cur)); - if (rl.rlim_cur != RLIM_INFINITY && rl.rlim_cur < 100 * 1024 * 1024) { - struct rlimit rl_new = { RLIM_INFINITY, RLIM_INFINITY }; - if (setrlimit(RLIMIT_MEMLOCK, &rl_new) == 0) { - rl.rlim_cur = RLIM_INFINITY; - rl.rlim_max = RLIM_INFINITY; - validation_json["memlock_limit"] = "infinity"; - if (print_human) - header_print_g("Linux", "Memlock Limit: set to infinity"); - } else { - if (print_human) { - header_print_r("ERROR", "Memlock limit is too low (" << (rl.rlim_cur / 1024 / 1024) << "MB). Please raise the limit or set to infinity."); - } - memlock_ok = false; - } - } else if (print_human) { - if (rl.rlim_cur == RLIM_INFINITY) { - header_print_g("Linux", "Memlock Limit: infinity"); - } else { - header_print("Linux", "Memlock Limit: " << (rl.rlim_cur / 1024 / 1024) << " MB"); - } - } - } else { - if (print_human) - perror("Failed to get memlock limit"); - } - validation_json["memlock_ok"] = memlock_ok; + const bool memlock_ok = check_memlock(validation_json, print_human); if (print_human) { header_print_g("Linux", "NPU platform: " << validation_json["npu_platform"].get());