From 5f1f3871ce5a6d6223d99ead46b55d6e4d17f553 Mon Sep 17 00:00:00 2001 From: Filipe Martins <293984334+filipemartinsubrobotics@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:12:27 +0100 Subject: [PATCH 1/2] fix: pass the CUDA toolkit include path to the XQA NVRTC JIT buildNvrtcOptions() passed no -I, on the grounds that every header reaches nvrtcCreateProgram as a virtual include. That holds for the kernel's own headers, but mha.cu includes cuda_fp16.h, which includes vector_types.h from the CUDA toolkit. NVRTC on CUDA 12.x for x86 does not carry that as a builtin, so every model whose decoder takes the XQA JIT path fails to load: cuda_fp16.h(129): catastrophic error: cannot open source file "vector_types.h" It surfaces during ONNX parsing as "Could not create the plugin", which points away from the cause. Pass one include path, taken from CUDA_DIR, which CMake already resolves for this build. EDGELLM_NVRTC_INCLUDE overrides it at runtime for installs relocated away from the toolkit they were built against, and the conventional install path is the fallback when the definition is absent. The option builder is shared by every SM, so this is not architecture specific. Verified on x86-64 with CUDA 12.8 against an SM89 GPU: the XQA kernel compiles in about a second, and a Cosmos3-Edge INT4-AWQ checkpoint then runs end to end with the environment variable unset, so the build-system path is the only source of the include directory. Fixes #204 Signed-off-by: Filipe Martins <293984334+filipemartinsubrobotics@users.noreply.github.com> --- cpp/CMakeLists.txt | 6 ++++++ .../decoderXQAJitCompiler.cpp | 20 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1b4840de9..58e8c5ad5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -182,6 +182,12 @@ target_include_directories(edgellmPluginJit PRIVATE ${COMMON_INCLUDE_DIRS} ${NVRTC_INCLUDE_DIR}) target_link_libraries( edgellmPluginJit PRIVATE ${NVRTC_LIB} ${CUDA_DRIVER_LINK_LIB} ${CUDART_LIB}) +# The XQA JIT compiles its kernel through NVRTC at load time and needs the +# toolkit's own headers on the include path (cuda_fp16.h -> vector_types.h). +# Pass the directory this build was configured against; EDGELLM_NVRTC_INCLUDE +# overrides it at runtime. +target_compile_definitions( + edgellmPluginJit PRIVATE EDGELLM_CUDA_INCLUDE_DIR="${CUDA_DIR}/include") add_dependencies(edgellmPluginJit generatePluginJitEmbeddedSources) edgellm_apply_qnx_warning_suppressions(edgellmPluginJit) edgellm_set_hidden_visibility(edgellmPluginJit) diff --git a/cpp/kernels/decodeAttentionKernels/decoderXQAJitCompiler.cpp b/cpp/kernels/decodeAttentionKernels/decoderXQAJitCompiler.cpp index 9922199c9..1127b7c33 100644 --- a/cpp/kernels/decodeAttentionKernels/decoderXQAJitCompiler.cpp +++ b/cpp/kernels/decodeAttentionKernels/decoderXQAJitCompiler.cpp @@ -24,11 +24,18 @@ #include #include +#include #include #include #include #include +// Set by the build system to the CUDA toolkit this build was configured against. The +// conventional install path is the fallback for builds that do not define it. +#ifndef EDGELLM_CUDA_INCLUDE_DIR +#define EDGELLM_CUDA_INCLUDE_DIR "/usr/local/cuda/include" +#endif + namespace trt_edgellm { namespace @@ -105,7 +112,18 @@ std::vector buildNvrtcOptions(XQAJitKey const& key) options.emplace_back("--use_fast_math"); options.emplace_back("--device-as-default-execution-space"); options.emplace_back(getGpuArchitectureOption(key.sm)); - // No -I flags needed: all headers are passed as virtual includes to nvrtcCreateProgram. + // The kernel's own headers are virtual includes on nvrtcCreateProgram, but cuda_fp16.h + // includes vector_types.h from the CUDA toolkit, which NVRTC does not carry as a builtin + // on CUDA 12.x for x86. Without this include path the compile fails with + // cuda_fp16.h(129): catastrophic error: cannot open source file "vector_types.h" + // and it surfaces during ONNX parsing as a plugin creation failure, which points away + // from the cause. EDGELLM_NVRTC_INCLUDE overrides the configured path for installs that + // are relocated away from the toolkit they were built against. + { + char const* const cudaIncludeOverride = std::getenv("EDGELLM_NVRTC_INCLUDE"); + options.emplace_back( + std::string("-I") + (cudaIncludeOverride != nullptr ? cudaIncludeOverride : EDGELLM_CUDA_INCLUDE_DIR)); + } options.emplace_back("-DGENERATE_CUBIN=1"); options.emplace_back("-DNDEBUG"); options.emplace_back("-DINFINITY=__int_as_float(0x7f800000)"); From 4e5edfff8c3cf01f4a5d2ee0f235646f1c34c7dc Mon Sep 17 00:00:00 2001 From: Filipe Martins <293984334+filipemartinsubrobotics@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:07:23 +0100 Subject: [PATCH 2/2] fix: resolve the XQA JIT include path from the NVRTC root, not just CUDA_DIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #205: FindNVRTC.cmake supports selecting NVRTC through NVRTC_ROOT_DIR or CUDA_TARGET_DIR, so baking ${CUDA_DIR}/include could link one toolkit's NVRTC while the runtime JIT searched another's headers. Resolving NVRTC_INCLUDE_DIR instead would not be right either: that is found by locating nvrtc.h, and a standalone NVRTC package ships nvrtc.h alone. The nvidia-cuda-nvrtc wheel's include/ holds exactly one header, so pointing NVRTC_ROOT_DIR at it and baking that directory would hand the JIT a tree with neither cuda_fp16.h nor vector_types.h — the failure this series fixes. So resolve the header that is actually included, across the roots FindNVRTC already honours and in the same precedence. A non-default NVRTC root wins when it can serve the compile and is declined when it cannot, with a warning and the CUDA_DIR fallback rather than a silent mismatch. Checked against four real configurations on one box: an nvidia-cu13 wheel with a full header set resolves to itself; NVRTC_ROOT_DIR=/usr with apt CUDA 11.5 there resolves to /usr/include; CUDA_TARGET_DIR at a second toolkit resolves to that toolkit; and the nvidia-cuda-nvrtc wheel falls back to CUDA_DIR with a warning. A cross build still bakes a path correct on the build host, which may not be where the toolkit sits on the target. EDGELLM_NVRTC_INCLUDE covers that, and a comment now says so. Signed-off-by: Filipe Martins <293984334+filipemartinsubrobotics@users.noreply.github.com> --- cpp/CMakeLists.txt | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58e8c5ad5..798508368 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -182,12 +182,31 @@ target_include_directories(edgellmPluginJit PRIVATE ${COMMON_INCLUDE_DIRS} ${NVRTC_INCLUDE_DIR}) target_link_libraries( edgellmPluginJit PRIVATE ${NVRTC_LIB} ${CUDA_DRIVER_LINK_LIB} ${CUDART_LIB}) -# The XQA JIT compiles its kernel through NVRTC at load time and needs the -# toolkit's own headers on the include path (cuda_fp16.h -> vector_types.h). -# Pass the directory this build was configured against; EDGELLM_NVRTC_INCLUDE -# overrides it at runtime. +# The XQA JIT compiles its kernel through NVRTC at load time and needs the CUDA +# runtime headers on the include path (cuda_fp16.h -> vector_types.h). Those are +# not necessarily where nvrtc.h is: in a full toolkit they sit together, but a +# standalone NVRTC package ships nvrtc.h alone. Search for the header that is +# actually included, across the roots FindNVRTC already honours and in the same +# precedence, so a non-default NVRTC_ROOT_DIR or CUDA_TARGET_DIR cannot leave +# the JIT reading a different toolkit's headers than the one it links against. +find_path( + EDGELLM_JIT_CUDA_INCLUDE_DIR vector_types.h + HINTS ${NVRTC_INCLUDE_DIR} ${NVRTC_ROOT_DIR}/include + ${CUDA_TARGET_DIR}/include ${CUDA_DIR}/include + NO_DEFAULT_PATH) +if(NOT EDGELLM_JIT_CUDA_INCLUDE_DIR) + message( + WARNING + "No vector_types.h under the NVRTC/CUDA roots; the XQA JIT will fall back " + "to ${CUDA_DIR}/include. Set NVRTC_ROOT_DIR or CUDA_DIR to a CUDA Toolkit " + "with the runtime headers, or set EDGELLM_NVRTC_INCLUDE at run time.") + set(EDGELLM_JIT_CUDA_INCLUDE_DIR "${CUDA_DIR}/include") +endif() +# Cross builds bake a path that is correct on the BUILD host; where the toolkit +# sits elsewhere on the target, EDGELLM_NVRTC_INCLUDE overrides it at run time. target_compile_definitions( - edgellmPluginJit PRIVATE EDGELLM_CUDA_INCLUDE_DIR="${CUDA_DIR}/include") + edgellmPluginJit + PRIVATE EDGELLM_CUDA_INCLUDE_DIR="${EDGELLM_JIT_CUDA_INCLUDE_DIR}") add_dependencies(edgellmPluginJit generatePluginJitEmbeddedSources) edgellm_apply_qnx_warning_suppressions(edgellmPluginJit) edgellm_set_hidden_visibility(edgellmPluginJit)