From ae26cd35f21dff43fdaa01c59a1ebe9425149c72 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 7 Sep 2026 07:30:58 -0700 Subject: [PATCH 1/3] lib: add fallback aliases for disabled BMI2 variants When dynamic BMI2 dispatch is disabled, the BMI2-specific functions are not compiled and each selector is conditionally compiled to avoid naming them. The selector-level preprocessor guards will be replaced with a predicate that becomes constant false when dynamic BMI2 dispatch is disabled. Although the compiler eliminates an if (0) branch, it must still parse and resolve the BMI2 function referenced by it. Add aliases from the unavailable BMI2 function names to their default implementations. These aliases make the names valid without emitting BMI2-specific code. The selectors remain unchanged in this patch, so the aliases are not used yet and there is no code-generation change. Signed-off-by: Usama Arif --- lib/common/entropy_common.c | 4 ++++ lib/common/fse_decompress.c | 2 ++ lib/compress/zstd_compress_sequences.c | 4 ++++ lib/decompress/huf_decompress.c | 4 ++++ lib/decompress/zstd_decompress_block.c | 12 ++++++++++++ 5 files changed, 26 insertions(+) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index e2173afb0a8..d73c8f00bae 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -201,6 +201,8 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( { return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); } +#else +#define FSE_readNCount_body_bmi2 FSE_readNCount_body_default #endif size_t FSE_readNCount_bmi2( @@ -322,6 +324,8 @@ static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, si { return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); } +#else +#define HUF_readStats_body_bmi2 HUF_readStats_body_default #endif size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index c8f1bb0cf23..0da22a25bbf 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -299,6 +299,8 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, siz { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } +#else +#define FSE_decompress_wksp_body_bmi2 FSE_decompress_wksp_body_default #endif size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index 7beb9daa603..2968a910996 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -414,6 +414,10 @@ ZSTD_encodeSequences_bmi2( sequences, nbSeq, longOffsets); } +#else + +#define ZSTD_encodeSequences_bmi2 ZSTD_encodeSequences_default + #endif size_t ZSTD_encodeSequences( diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 92020617824..ae86ba18b8f 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -704,6 +704,8 @@ size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } +#else +#define HUF_decompress4X1_usingDTable_internal_bmi2 HUF_decompress4X1_usingDTable_internal_default #endif static @@ -1507,6 +1509,8 @@ size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } +#else +#define HUF_decompress4X2_usingDTable_internal_bmi2 HUF_decompress4X2_usingDTable_internal_default #endif static diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 56e1f9eda28..4087ab7ebc7 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -620,6 +620,8 @@ BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* d ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); } +#else +#define ZSTD_buildFSETable_body_bmi2 ZSTD_buildFSETable_body_default #endif void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, @@ -2035,6 +2037,16 @@ ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +#else + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +#define ZSTD_decompressSequences_bmi2 ZSTD_decompressSequences_default +#define ZSTD_decompressSequencesSplitLitBuffer_bmi2 ZSTD_decompressSequencesSplitLitBuffer_default +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +#define ZSTD_decompressSequencesLong_bmi2 ZSTD_decompressSequencesLong_default +#endif + #endif /* DYNAMIC_BMI2 */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG From 86cb063c27418d67c777c4814c9f3cf2383712f9 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 7 Sep 2026 07:31:24 -0700 Subject: [PATCH 2/3] lib: allow integrations to override BMI2 dispatch Dynamic BMI2 dispatch probes CPUID when a compression or decompression context is initialized. It caches the result in the context and tests the caller-provided state at each final dispatch site. This is suitable for standalone userspace, but prevents integrations such as the Linux kernel from applying their own CPU feature policy at those sites. Add ZSTD_USE_BMI2() for final selectors and ZSTD_SET_BMI2() for context initialization. Their default definitions preserve existing userspace behavior, while allowing an integration to replace both the selection policy and cached state. For the Linux kernel import, normal x86 objects now select BMI2 through cpu_feature_enabled(X86_FEATURE_BMI2), allowing x86 alternatives to resolve the check. Preboot objects, identified by __DISABLE_EXPORTS, retain the existing CPUID-backed path because the normal kernel CPU feature infrastructure is unavailable there. Use the fallback aliases added by the preceding change when target- attributed variants are absent, and verify that the import process consumes ZSTD_LINUX_KERNEL. A 4 KiB zstd-generic crypto_acomp benchmark in a one-vCPU KVM guest gave these median results: Before After Change Compression 16,634 ns/op 13,394 ns/op -19.5% Decompression 3,480 ns/op 963 ns/op -72.3% Signed-off-by: Usama Arif --- contrib/linux-kernel/test/macro-test.sh | 1 + lib/BUCK | 1 + lib/common/bmi2.h | 40 +++++++++++++++++++++++ lib/common/entropy_common.c | 9 ++--- lib/common/fse_decompress.c | 5 ++- lib/common/portability_macros.h | 12 +++++++ lib/compress/huf_compress.c | 4 ++- lib/compress/zstd_compress.c | 5 +-- lib/compress/zstd_compress_internal.h | 15 +++++---- lib/compress/zstd_compress_sequences.c | 5 ++- lib/decompress/huf_decompress.c | 16 ++++----- lib/decompress/zstd_decompress.c | 3 +- lib/decompress/zstd_decompress_block.c | 17 +++------- lib/decompress/zstd_decompress_internal.h | 15 +++++---- 14 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 lib/common/bmi2.h diff --git a/contrib/linux-kernel/test/macro-test.sh b/contrib/linux-kernel/test/macro-test.sh index 9ea84aa66b1..403870561a0 100755 --- a/contrib/linux-kernel/test/macro-test.sh +++ b/contrib/linux-kernel/test/macro-test.sh @@ -35,6 +35,7 @@ println "" test_not_present "ZSTD_NO_INTRINSICS" test_not_present "ZSTD_NO_UNUSED_FUNCTIONS" test_not_present "ZSTD_LEGACY_SUPPORT" +test_not_present "ZSTD_LINUX_KERNEL" test_not_present "STATIC_BMI2" test_not_present "ZSTD_DLL_EXPORT" test_not_present "ZSTD_DLL_IMPORT" diff --git a/lib/BUCK b/lib/BUCK index 8c555a8be18..6541069d383 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -79,6 +79,7 @@ cxx_library( header_namespace='', visibility=['PUBLIC'], exported_headers=subdir_glob([ + ('common', 'bmi2.h'), ('common', 'compiler.h'), ]), ) diff --git a/lib/common/bmi2.h b/lib/common/bmi2.h new file mode 100644 index 00000000000..8e61e23e253 --- /dev/null +++ b/lib/common/bmi2.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BMI2_H +#define ZSTD_BMI2_H + +#include "portability_macros.h" + +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +# include +#endif + +/* + * Select between the BMI2 and default implementations at the final dispatch + * site. Integrations may ignore the caller-provided hint and consult their own + * CPU-feature policy instead. + * + * ZSTD_USE_BMI2() may not evaluate its argument. ZSTD_SET_BMI2() may not + * evaluate its value argument, which avoids executing the userspace CPUID + * probe when the integration supplies the dispatch decision. + */ +#if !DYNAMIC_BMI2 +# define ZSTD_USE_BMI2(bmi2) 0 +# define ZSTD_SET_BMI2(state, value) do { } while (0) +#elif defined(ZSTD_USE_KERNEL_CPU_FEATURES) +# define ZSTD_USE_BMI2(bmi2) cpu_feature_enabled(X86_FEATURE_BMI2) +# define ZSTD_SET_BMI2(state, value) do { (state) = 0; } while (0) +#else +# define ZSTD_USE_BMI2(bmi2) (bmi2) +# define ZSTD_SET_BMI2(state, value) do { (state) = (value); } while (0) +#endif + +#endif /* ZSTD_BMI2_H */ diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index d73c8f00bae..f03a3fabbcc 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -15,6 +15,7 @@ /* ************************************* * Dependencies ***************************************/ +#include "bmi2.h" #include "mem.h" #include "error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ @@ -209,11 +210,9 @@ size_t FSE_readNCount_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); } -#endif (void)bmi2; return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); } @@ -334,11 +333,9 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, void* workSpace, size_t wkspSize, int flags) { -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } -#endif (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 0da22a25bbf..521f758a594 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -18,6 +18,7 @@ ****************************************************************/ #include "debug.h" /* assert */ #include "bitstream.h" +#include "bmi2.h" #include "compiler.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" @@ -305,11 +306,9 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, siz size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } -#endif (void)bmi2; return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h index bcca634e419..857822f91eb 100644 --- a/lib/common/portability_macros.h +++ b/lib/common/portability_macros.h @@ -102,6 +102,18 @@ # endif #endif +/* + * The Linux kernel importer consumes the outer guard while preserving the + * distinction below. Normal x86 kernel objects can use the kernel CPU-feature + * policy, while preboot objects cannot use that infrastructure. + */ +#if defined(ZSTD_LINUX_KERNEL) +# if defined(__KERNEL__) && defined(CONFIG_X86) && \ + !defined(__DISABLE_EXPORTS) +# define ZSTD_USE_KERNEL_CPU_FEATURES +# endif +#endif + /** * Only enable assembly for GNU C compatible compilers, * because other platforms may not support GAS assembly syntax. diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 6f2194e7934..009182725c0 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -24,6 +24,7 @@ * Includes ****************************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/bmi2.h" #include "../common/compiler.h" #include "../common/bitstream.h" #include "hist.h" @@ -1141,9 +1142,10 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, const int flags) { - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); } + (void)flags; return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 50c3e6f7279..38c3fc0d4cd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,6 +13,7 @@ ***************************************/ #include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/bmi2.h" #include "../common/mem.h" #include "../common/error_private.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -113,7 +114,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) ZSTD_memset(cctx, 0, sizeof(*cctx)); cctx->customMem = memManager; #if DYNAMIC_BMI2 - cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuSupportsBmi2()); #endif { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); assert(!ZSTD_isError(err)); @@ -155,7 +156,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; #if DYNAMIC_BMI2 - cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuSupportsBmi2()); #endif return cctx; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index e233bc5ff1b..c15a94da3b5 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -473,9 +473,10 @@ struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ #if DYNAMIC_BMI2 - int bmi2; /* == 1 if the CPU supports BMI1 & BMI2, determined once per context lifetime. - * Never read this directly: use ZSTD_CCtx_get_bmi2(), which also handles builds - * where BMI2 is enabled at compile time and this field does not exist. */ + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise when CPU state + * is cached per context. Never read this directly: use ZSTD_CCtx_get_bmi2(), + * which also handles builds + * where cached CPU state is not used. */ #endif ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; @@ -550,13 +551,15 @@ struct ZSTD_CCtx_s { }; /** - * @returns 1 if calls should be dispatched to the BMI2_TARGET_ATTRIBUTE variant - * of a function, rather than to its default variant. + * @returns the cached per-context BMI2 dispatch hint, or 0 when the build does + * not use cached CPU state. * * Beware: this is *not* a "does the CPU support BMI2" test. When BMI2 is enabled * at compile time, DYNAMIC_BMI2 is 0: the whole library is already compiled with * BMI2, no separate variant exists, and this returns 0 even though the CPU does - * support BMI2. Only ever use it to select between the two variants of a function. + * support BMI2. Only use the result as a dispatch hint passed to a lower-level + * BMI2/default selector. Integrations may ignore the cached hint and consult + * their own CPU-feature policy at that selector. */ MEM_STATIC int ZSTD_CCtx_get_bmi2(const struct ZSTD_CCtx_s* cctx) { #if DYNAMIC_BMI2 diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index 2968a910996..a7769cba1f3 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -11,6 +11,7 @@ /*-************************************* * Dependencies ***************************************/ +#include "../common/bmi2.h" #include "zstd_compress_sequences.h" /** @@ -428,15 +429,13 @@ size_t ZSTD_encodeSequences( SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) { DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return ZSTD_encodeSequences_bmi2(dst, dstCapacity, CTable_MatchLength, mlCodeTable, CTable_OffsetBits, ofCodeTable, CTable_LitLength, llCodeTable, sequences, nbSeq, longOffsets); } -#endif (void)bmi2; return ZSTD_encodeSequences_default(dst, dstCapacity, CTable_MatchLength, mlCodeTable, diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index ae86ba18b8f..1cf329aae75 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -17,6 +17,7 @@ ****************************************************************/ #include /* size_t */ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/bmi2.h" #include "../common/compiler.h" #include "../common/bitstream.h" /* BIT_* */ #include "../common/fse.h" /* to compress headers */ @@ -117,9 +118,10 @@ typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - if (flags & HUF_flags_bmi2) { \ + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { \ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ + (void)flags; \ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ } @@ -903,18 +905,16 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 if (!(flags & HUF_flags_disableAsm)) { loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; } # endif - } else { + } else if (DYNAMIC_BMI2) { return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) if (!(flags & HUF_flags_disableAsm)) { @@ -1735,18 +1735,16 @@ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 if (!(flags & HUF_flags_disableAsm)) { loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; } # endif - } else { + } else if (DYNAMIC_BMI2) { return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) if (!(flags & HUF_flags_disableAsm)) { diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 7e6e01b4759..325f0b54dfc 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -57,6 +57,7 @@ *********************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/bmi2.h" #include "../common/error_private.h" #include "../common/zstd_internal.h" /* blockProperties_t */ #include "../common/mem.h" /* low level memory routines */ @@ -267,7 +268,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->oversizedDuration = 0; dctx->isFrameDecompression = 1; #if DYNAMIC_BMI2 - dctx->bmi2 = ZSTD_cpuSupportsBmi2(); + ZSTD_SET_BMI2(dctx->bmi2, ZSTD_cpuSupportsBmi2()); #endif dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 4087ab7ebc7..34a22394f7b 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -15,6 +15,7 @@ * Dependencies *********************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/bmi2.h" #include "../common/compiler.h" /* prefetch */ #include "../common/mem.h" /* low level memory routines */ #include @@ -629,13 +630,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); return; } -#endif (void)bmi2; ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); @@ -2056,11 +2055,9 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } static size_t @@ -2069,11 +2066,9 @@ ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDst const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -2092,11 +2087,9 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h index 07b9732b4d4..58a5bc8cc8f 100644 --- a/lib/decompress/zstd_decompress_internal.h +++ b/lib/decompress/zstd_decompress_internal.h @@ -155,9 +155,10 @@ struct ZSTD_DCtx_s size_t staticSize; int isFrameDecompression; #if DYNAMIC_BMI2 - int bmi2; /* == 1 if the CPU supports BMI1 & BMI2, determined once per context lifetime. - * Never read this directly: use ZSTD_DCtx_get_bmi2(), which also handles builds - * where BMI2 is enabled at compile time and this field does not exist. */ + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise when CPU state + * is cached per context. Never read this directly: use ZSTD_DCtx_get_bmi2(), + * which also handles builds + * where cached CPU state is not used. */ #endif /* dictionary */ @@ -213,13 +214,15 @@ struct ZSTD_DCtx_s }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ /** - * @returns 1 if calls should be dispatched to the BMI2_TARGET_ATTRIBUTE variant - * of a function, rather than to its default variant. + * @returns the cached per-context BMI2 dispatch hint, or 0 when the build does + * not use cached CPU state. * * Beware: this is *not* a "does the CPU support BMI2" test. When BMI2 is enabled * at compile time, DYNAMIC_BMI2 is 0: the whole library is already compiled with * BMI2, no separate variant exists, and this returns 0 even though the CPU does - * support BMI2. Only ever use it to select between the two variants of a function. + * support BMI2. Only use the result as a dispatch hint passed to a lower-level + * BMI2/default selector. Integrations may ignore the cached hint and consult + * their own CPU-feature policy at that selector. */ MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { #if DYNAMIC_BMI2 From 8db0c1376d618a7ca064584a4a1c92c7c28a8406 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 11 Sep 2026 03:43:35 -0700 Subject: [PATCH 3/3] lib: test kernel BMI2 feature dispatch The linux-kernel test harness compiles imported sources without __KERNEL__ or CONFIG_X86, so it does not exercise the ZSTD_USE_KERNEL_CPU_FEATURES path. Add a mock asm/cpufeature.h and a second test variant that compiles imported objects with __KERNEL__ and CONFIG_X86. Keep the objects in a separate tree so parallel builds do not reuse objects built with different flags. Signed-off-by: Usama Arif --- contrib/linux-kernel/test/Makefile | 28 +++++++++++++++++-- .../test/include/asm/cpufeature.h | 16 +++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 contrib/linux-kernel/test/include/asm/cpufeature.h diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 67b55e66504..f7a48773912 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -23,27 +23,51 @@ LINUX_ZSTD_FILES := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_ LINUX_ZSTD_OBJECTS0 := $(LINUX_ZSTD_FILES:.c=.o) LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_OBJECTS0:.S=.o) +X86_BUILD_DIR := x86 +X86_LINUX_ZSTD_FILES := $(patsubst $(LINUX_ZSTDLIB)/%,%,$(LINUX_ZSTD_FILES)) +X86_LINUX_ZSTD_OBJECTS0 := $(patsubst %.c,$(X86_BUILD_DIR)/%.o,$(X86_LINUX_ZSTD_FILES)) +X86_LINUX_ZSTD_OBJECTS := $(patsubst %.S,$(X86_BUILD_DIR)/%.o,$(X86_LINUX_ZSTD_OBJECTS0)) +X86_LINUX_ZSTD_CPPFLAGS := -D__KERNEL__ -DCONFIG_X86 + %.o: %.S $(COMPILE.S) $(OUTPUT_OPTION) $< +$(X86_LINUX_ZSTD_OBJECTS): CPPFLAGS += $(X86_LINUX_ZSTD_CPPFLAGS) + +$(X86_BUILD_DIR)/%.o: $(LINUX_ZSTDLIB)/%.c + @mkdir -p $(@D) + $(COMPILE.c) $(OUTPUT_OPTION) $< + +$(X86_BUILD_DIR)/%.o: $(LINUX_ZSTDLIB)/%.S + @mkdir -p $(@D) + $(COMPILE.S) $(OUTPUT_OPTION) $< + liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS) $(AR) $(ARFLAGS) $@ $^ +liblinuxzstd_x86.a: $(X86_LINUX_ZSTD_OBJECTS) + $(AR) $(ARFLAGS) $@ $^ + test: test.c liblinuxzstd.a $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@ +test_x86: test.c liblinuxzstd_x86.a + $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@ + static_test: static_test.c $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -run-test: test static_test +run-test: test test_x86 static_test ./macro-test.sh ./test + ./test_x86 ./static_test .PHONY: clean: $(RM) -f $(LINUX_ZSTDLIB)/*.o $(RM) -f $(LINUX_ZSTDLIB)/**/*.o + $(RM) -rf $(X86_BUILD_DIR) $(RM) -f *.o *.a $(RM) -f static_test - $(RM) -f test + $(RM) -f test test_x86 diff --git a/contrib/linux-kernel/test/include/asm/cpufeature.h b/contrib/linux-kernel/test/include/asm/cpufeature.h new file mode 100644 index 00000000000..7ac74137710 --- /dev/null +++ b/contrib/linux-kernel/test/include/asm/cpufeature.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef ASM_CPUFEATURE_H +#define ASM_CPUFEATURE_H + +#define X86_FEATURE_BMI2 0 +#define cpu_feature_enabled(feature) ((void)(feature), 0) + +#endif