terrelln · Sep 14, 2026 · Sep 7, 2026 · Sep 7, 2026 · Sep 11, 2026 · Cyan4973
diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile
@@ -23,27 +23,51 @@ LINUX_ZSTD_FILES      := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_
 LINUX_ZSTD_OBJECTS0   := $(LINUX_ZSTD_FILES:.c=.o)
 LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_OBJECTS0:.S=.o)
 
+X86_BUILD_DIR              := x86
+X86_LINUX_ZSTD_FILES       := $(patsubst $(LINUX_ZSTDLIB)/%,%,$(LINUX_ZSTD_FILES))
+X86_LINUX_ZSTD_OBJECTS0    := $(patsubst %.c,$(X86_BUILD_DIR)/%.o,$(X86_LINUX_ZSTD_FILES))
+X86_LINUX_ZSTD_OBJECTS     := $(patsubst %.S,$(X86_BUILD_DIR)/%.o,$(X86_LINUX_ZSTD_OBJECTS0))
+X86_LINUX_ZSTD_CPPFLAGS    := -D__KERNEL__ -DCONFIG_X86
+
 %.o: %.S
 	$(COMPILE.S) $(OUTPUT_OPTION) $<
 
+$(X86_LINUX_ZSTD_OBJECTS): CPPFLAGS += $(X86_LINUX_ZSTD_CPPFLAGS)
+
+$(X86_BUILD_DIR)/%.o: $(LINUX_ZSTDLIB)/%.c
+	@mkdir -p $(@D)
+	$(COMPILE.c) $(OUTPUT_OPTION) $<
+
+$(X86_BUILD_DIR)/%.o: $(LINUX_ZSTDLIB)/%.S
+	@mkdir -p $(@D)
+	$(COMPILE.S) $(OUTPUT_OPTION) $<
+
 liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
 	$(AR) $(ARFLAGS) $@ $^
 
+liblinuxzstd_x86.a: $(X86_LINUX_ZSTD_OBJECTS)
+	$(AR) $(ARFLAGS) $@ $^
+
 test: test.c liblinuxzstd.a
 	$(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
 
+test_x86: test.c liblinuxzstd_x86.a
+	$(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
+
 static_test: static_test.c
 	$(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
 
-run-test: test static_test
+run-test: test test_x86 static_test
 	./macro-test.sh
 	./test
+	./test_x86
 	./static_test
 
 .PHONY:
 clean:
 	$(RM) -f $(LINUX_ZSTDLIB)/*.o
 	$(RM) -f $(LINUX_ZSTDLIB)/**/*.o
+	$(RM) -rf $(X86_BUILD_DIR)
 	$(RM) -f *.o *.a
 	$(RM) -f static_test
-	$(RM) -f test
+	$(RM) -f test test_x86
diff --git a/contrib/linux-kernel/test/include/asm/cpufeature.h b/contrib/linux-kernel/test/include/asm/cpufeature.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef ASM_CPUFEATURE_H
+#define ASM_CPUFEATURE_H
+
+#define X86_FEATURE_BMI2 0
+#define cpu_feature_enabled(feature) ((void)(feature), 0)
+
+#endif
diff --git a/contrib/linux-kernel/test/macro-test.sh b/contrib/linux-kernel/test/macro-test.sh
@@ -35,6 +35,7 @@ println ""
 test_not_present "ZSTD_NO_INTRINSICS"
 test_not_present "ZSTD_NO_UNUSED_FUNCTIONS"
 test_not_present "ZSTD_LEGACY_SUPPORT"
+test_not_present "ZSTD_LINUX_KERNEL"
 test_not_present "STATIC_BMI2"
 test_not_present "ZSTD_DLL_EXPORT"
 test_not_present "ZSTD_DLL_IMPORT"

diff --git a/lib/BUCK b/lib/BUCK
@@ -79,6 +79,7 @@ cxx_library(
     header_namespace='',
     visibility=['PUBLIC'],
     exported_headers=subdir_glob([
+        ('common', 'bmi2.h'),
         ('common', 'compiler.h'),
     ]),
 )

diff --git a/lib/common/bmi2.h b/lib/common/bmi2.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_BMI2_H
+#define ZSTD_BMI2_H
+
+#include "portability_macros.h"
+
+#if defined(ZSTD_USE_KERNEL_CPU_FEATURES)
+#  include <asm/cpufeature.h>
+#endif
+
+/*
+ * Select between the BMI2 and default implementations at the final dispatch
+ * site. Integrations may ignore the caller-provided hint and consult their own
+ * CPU-feature policy instead.
+ *
+ * ZSTD_USE_BMI2() may not evaluate its argument. ZSTD_SET_BMI2() may not
+ * evaluate its value argument, which avoids executing the userspace CPUID
+ * probe when the integration supplies the dispatch decision.
+ */
+#if !DYNAMIC_BMI2
+#  define ZSTD_USE_BMI2(bmi2) 0
+#  define ZSTD_SET_BMI2(state, value) do { } while (0)
+#elif defined(ZSTD_USE_KERNEL_CPU_FEATURES)
+#  define ZSTD_USE_BMI2(bmi2) cpu_feature_enabled(X86_FEATURE_BMI2)
+#  define ZSTD_SET_BMI2(state, value) do { (state) = 0; } while (0)
+#else
+#  define ZSTD_USE_BMI2(bmi2) (bmi2)
+#  define ZSTD_SET_BMI2(state, value) do { (state) = (value); } while (0)
+#endif
+
+#endif /* ZSTD_BMI2_H */
diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
@@ -15,6 +15,7 @@
 /* *************************************
 *  Dependencies
 ***************************************/
+#include "bmi2.h"
 #include "mem.h"
 #include "error_private.h"       /* ERR_*, ERROR */
 #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
@@ -201,17 +202,17 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
 {
     return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
 }
+#else
+#define FSE_readNCount_body_bmi2 FSE_readNCount_body_default
 #endif
 
 size_t FSE_readNCount_bmi2(
         short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
         const void* headerBuffer, size_t hbSize, int bmi2)
 {
-#if DYNAMIC_BMI2
-    if (bmi2) {
+    if (ZSTD_USE_BMI2(bmi2)) {
         return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
     }
-#endif
     (void)bmi2;
     return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
 }
@@ -322,6 +323,8 @@ static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, si
 {
     return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
 }
+#else
+#define HUF_readStats_body_bmi2 HUF_readStats_body_default
 #endif
 
 size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
@@ -330,11 +333,9 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      void* workSpace, size_t wkspSize,
                      int flags)
 {
-#if DYNAMIC_BMI2
-    if (flags & HUF_flags_bmi2) {
+    if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) {
         return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
     }
-#endif
     (void)flags;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
@@ -18,6 +18,7 @@
 ****************************************************************/
 #include "debug.h"      /* assert */
 #include "bitstream.h"
+#include "bmi2.h"
 #include "compiler.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
@@ -299,15 +300,15 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, siz
 {
     return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
 }
+#else
+#define FSE_decompress_wksp_body_bmi2 FSE_decompress_wksp_body_default
 #endif
 
 size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
 {
-#if DYNAMIC_BMI2
-    if (bmi2) {
+    if (ZSTD_USE_BMI2(bmi2)) {
         return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
     }
-#endif
     (void)bmi2;
     return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
 }

diff --git a/lib/common/portability_macros.h b/lib/common/portability_macros.h
@@ -102,6 +102,18 @@
 #  endif
 #endif
 
+/*
+ * The Linux kernel importer consumes the outer guard while preserving the
+ * distinction below. Normal x86 kernel objects can use the kernel CPU-feature
+ * policy, while preboot objects cannot use that infrastructure.
+ */
+#if defined(ZSTD_LINUX_KERNEL)
+#  if defined(__KERNEL__) && defined(CONFIG_X86) && \
+      !defined(__DISABLE_EXPORTS)
+#    define ZSTD_USE_KERNEL_CPU_FEATURES
+#  endif
+#endif
+
 /**
  * Only enable assembly for GNU C compatible compilers,
  * because other platforms may not support GAS assembly syntax.

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
@@ -24,6 +24,7 @@
 *  Includes
 ****************************************************************/
 #include "../common/zstd_deps.h"     /* ZSTD_memcpy, ZSTD_memset */
+#include "../common/bmi2.h"
 #include "../common/compiler.h"
 #include "../common/bitstream.h"
 #include "hist.h"
@@ -1141,9 +1142,10 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
                               const HUF_CElt* CTable, const int flags)
 {
-    if (flags & HUF_flags_bmi2) {
+    if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) {
         return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
     }
+    (void)flags;
     return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
 }
 

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
@@ -13,6 +13,7 @@
 ***************************************/
 #include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"  /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
+#include "../common/bmi2.h"
 #include "../common/mem.h"
 #include "../common/error_private.h"
 #include "hist.h"           /* HIST_countFast_wksp */
@@ -113,7 +114,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager)
     ZSTD_memset(cctx, 0, sizeof(*cctx));
     cctx->customMem = memManager;
 #if DYNAMIC_BMI2
-    cctx->bmi2 = ZSTD_cpuSupportsBmi2();
+    ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuSupportsBmi2());
 #endif
     {   size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
         assert(!ZSTD_isError(err));
@@ -155,7 +156,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize)
     cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE);
     cctx->tmpWkspSize = TMP_WORKSPACE_SIZE;
 #if DYNAMIC_BMI2
-    cctx->bmi2 = ZSTD_cpuSupportsBmi2();
+    ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuSupportsBmi2());
 #endif
     return cctx;
 }

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
@@ -473,9 +473,10 @@ struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
     int cParamsChanged;                  /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
 #if DYNAMIC_BMI2
-    int bmi2;                            /* == 1 if the CPU supports BMI1 & BMI2, determined once per context lifetime.
-                                          * Never read this directly: use ZSTD_CCtx_get_bmi2(), which also handles builds
-                                          * where BMI2 is enabled at compile time and this field does not exist. */
+    int bmi2;                            /* == 1 if the CPU supports BMI2 and 0 otherwise when CPU state
+                                          * is cached per context. Never read this directly: use ZSTD_CCtx_get_bmi2(),
+                                          * which also handles builds
+                                          * where cached CPU state is not used. */
 #endif
     ZSTD_CCtx_params requestedParams;
     ZSTD_CCtx_params appliedParams;
@@ -550,13 +551,15 @@ struct ZSTD_CCtx_s {
 };
 
 /**
- * @returns 1 if calls should be dispatched to the BMI2_TARGET_ATTRIBUTE variant
- *          of a function, rather than to its default variant.
+ * @returns the cached per-context BMI2 dispatch hint, or 0 when the build does
+ *          not use cached CPU state.
  *
  * Beware: this is *not* a "does the CPU support BMI2" test. When BMI2 is enabled
  * at compile time, DYNAMIC_BMI2 is 0: the whole library is already compiled with
  * BMI2, no separate variant exists, and this returns 0 even though the CPU does
- * support BMI2. Only ever use it to select between the two variants of a function.
+ * support BMI2. Only use the result as a dispatch hint passed to a lower-level
+ * BMI2/default selector. Integrations may ignore the cached hint and consult
+ * their own CPU-feature policy at that selector.
  */
 MEM_STATIC int ZSTD_CCtx_get_bmi2(const struct ZSTD_CCtx_s* cctx) {
 #if DYNAMIC_BMI2

diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c
@@ -11,6 +11,7 @@
  /*-*************************************
  *  Dependencies
  ***************************************/
+#include "../common/bmi2.h"
 #include "zstd_compress_sequences.h"
 
 /**
@@ -414,6 +415,10 @@ ZSTD_encodeSequences_bmi2(
                                     sequences, nbSeq, longOffsets);
 }
 
+#else
+
+#define ZSTD_encodeSequences_bmi2 ZSTD_encodeSequences_default
+
 #endif
 
 size_t ZSTD_encodeSequences(
@@ -424,15 +429,13 @@ size_t ZSTD_encodeSequences(
             SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
 {
     DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
-#if DYNAMIC_BMI2
-    if (bmi2) {
+    if (ZSTD_USE_BMI2(bmi2)) {
         return ZSTD_encodeSequences_bmi2(dst, dstCapacity,
                                          CTable_MatchLength, mlCodeTable,
                                          CTable_OffsetBits, ofCodeTable,
                                          CTable_LitLength, llCodeTable,
                                          sequences, nbSeq, longOffsets);
     }
-#endif
     (void)bmi2;
     return ZSTD_encodeSequences_default(dst, dstCapacity,
                                         CTable_MatchLength, mlCodeTable,