From 9ad16f432ffcba63e4d651fa46ded8b90b85369d Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Wed, 4 Nov 2026 10:43:00 +1000 Subject: [PATCH] x86_64 asm: load 32-bit parameters as 32-bit and ArrowLake/LunarLake change On Windows x64, chacha20_poly1305_small_enc/_dec read the word32 mLen argument from the stack with a QWORD load. MSVC writes only the low dword of that slot, so the upper half came from the caller's stack contents and corrupted the length. Encrypting or decrypting <= 64 bytes crashed when that stack slot was non-zero. The SysV build was not affected. The same QWORD load of a 32-bit stack parameter is also fixed in chacha20_poly1305_avx512/_ifma/_ifma_decrypt, the SHA-3 LMS/XMSS x4/x8 block functions, FrodoKEM, and ML-KEM keygen/encapsulate/decapsulate. In the .S files, register moves of 32-bit parameters become 32-bit (movl), which zero-extends them. Every changed 32-bit parameter was checked against its C prototype: all are non-pointer 32-bit scalars. On Intel Arrow Lake the VAES AES code is no faster than the AVX2/AES-NI code, and the single-stream AVX2 SHA-3 block is slower than the BMI2 and C blocks. Lunar Lake uses the same cores and is treated the same way. Add two tuning bits to the Intel cpuid flags. They are not instruction set features: they mark a CPU where an implementation the feature bits allow is measured to be slower than a narrower one. CPUID_SLOW_VAES AES does not use VAES CPUID_SLOW_AVX2_SHA3 SHA-3 and ML-DSA do not use the AVX2 Keccak block cpuid.c sets both from the family/model on Intel family 6 models 0xB5, 0xC5, 0xC6 (Arrow Lake U/H/S) and 0xBD (Lunar Lake M). Define WC_CPUID_NO_TUNING to turn this off. The bits can also be changed at run time with cpuid_set_flag()/cpuid_clear_flag(). The CPUID_VAES and CPUID_AVX2 feature bits are unchanged, so FrodoKEM, ChaCha20 and Poly1305 dispatch as before. aes.c: all VAES selection goes through AES_USE_VAES(f), which is false when CPUID_SLOW_VAES is set. This covers ECB, CBC, CTR, GCM, XTS and AES-GCM-SIV (CTR and POLYVAL). Those CPUs fall back to the AVX2 / AVX1 / AES-NI code. sha3.h: SHA3_USE_AVX2() also requires !IS_CPU_SLOW_AVX2_SHA3(f), so ML-DSA, which selected the AVX2 block first on Intel, uses BMI2 instead. SHA-3 already preferred BMI2. WOLFSSL_SHA3_AVX2 still forces AVX2. The 4-way AVX2 Keccak and the ML-DSA AVX2 NTT/poly code are unchanged. chacha_asm.asm: regenerate. chacha_encrypt_x64 used r8/r9 for ChaCha state words 4 and 5, but on Windows x64 those registers hold the output pointer and length. The output was wrong and the partial-block tail wrote through a bad pointer. It crashed on AVX-512VL CPUs, which call this function for messages of 64 bytes or less (e.g. the TLS Poly1305 key). It now uses rdi/rsi, saved and restored. chacha_asm.S is unchanged. chacha.c: only let AVX-512VL steer short input when the AVX-512VL kernel is built (CHACHA_USE_AVX512_VL). With NO_AVX512_SUPPORT, input shorter than 256 bytes no longer skips the AVX1/SSSE3 paths for the scalar code. --- .wolfssl_known_macro_extras | 2 + tests/unit-mcdc/test_aes_whitebox.c | 442 ++++++++++++++++++++++- tests/unit-mcdc/test_chacha_whitebox.c | 13 +- tests/unit-mcdc/test_sha3_whitebox.c | 6 +- tests/unit-mcdc/test_wc_mldsa_whitebox.c | 13 +- wolfcrypt/benchmark/benchmark.c | 2 + wolfcrypt/src/aes.c | 123 ++++--- wolfcrypt/src/aes_gcm_asm.S | 57 +++ wolfcrypt/src/aes_gcm_asm.asm | 34 +- wolfcrypt/src/aes_x86_64_asm.S | 20 + wolfcrypt/src/aes_xts_asm.S | 44 ++- wolfcrypt/src/aes_xts_asm.asm | 36 +- wolfcrypt/src/chacha.c | 20 +- wolfcrypt/src/chacha_asm.S | 11 + wolfcrypt/src/chacha_asm.asm | 150 ++++---- wolfcrypt/src/cpuid.c | 44 +++ wolfcrypt/src/fe_x25519_asm.S | 1 + wolfcrypt/src/fe_x25519_asm.asm | 1 + wolfcrypt/src/sha256_asm.S | 14 +- wolfcrypt/src/sha256_asm.asm | 10 +- wolfcrypt/src/sha3_asm.S | 10 + wolfcrypt/src/sha3_asm.asm | 14 +- wolfcrypt/src/sha512_asm.S | 7 +- wolfcrypt/src/sha512_asm.asm | 8 +- wolfcrypt/src/wc_frodokem_asm.S | 39 +- wolfcrypt/src/wc_frodokem_asm.asm | 53 ++- wolfcrypt/src/wc_mldsa_asm.S | 22 +- wolfcrypt/src/wc_mldsa_asm.asm | 26 +- wolfcrypt/src/wc_mlkem_asm.S | 48 ++- wolfcrypt/src/wc_mlkem_asm.asm | 50 ++- wolfssl/wolfcrypt/cpuid.h | 17 + wolfssl/wolfcrypt/sha3.h | 10 +- 32 files changed, 1102 insertions(+), 245 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 6ebc459788e..612efddb1b2 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -835,6 +835,7 @@ WC_ASYNC_THREAD_BIND WC_BENCH_AES_IV_SWEEP WC_BLINDING_NO_RNG_ACKNOWLEDGE_WEAKNESS WC_CACHE_RESISTANT_BASE64_TABLE +WC_CPUID_NO_TUNING WC_DISABLE_RADIX_ZERO_PAD WC_DRBG_BANKREF WC_FIPS_ED25519CTX_NOT_APPROVED @@ -1506,6 +1507,7 @@ __ghc__ __ghs__ __has_attribute __has_include +__has_warning __hpux__ __i386 __i386__ diff --git a/tests/unit-mcdc/test_aes_whitebox.c b/tests/unit-mcdc/test_aes_whitebox.c index 6183d24d82b..830a75208d9 100644 --- a/tests/unit-mcdc/test_aes_whitebox.c +++ b/tests/unit-mcdc/test_aes_whitebox.c @@ -38,6 +38,9 @@ * Class 9 wc_AesCcmEncrypt()'s use_aesni dispatch (WOLFSSL_AESNI) 1 condition * Class 10 AArch64 CTR leftover-keystream loop (WOLFSSL_ARMASM, * __aarch64__, qemu-aarch64 lane only) .. exclusion demonstration + * Class 11 AES_USE_VAES / CPUID_SLOW_VAES fallback (x86_64 VAES + * builds): forced on any host, dispatch counted, outputs + * compared with the fallback ................. 2 conditions * Classes 4, 5 and 10 only compile in the qemu-aarch64 emulator lane (see * iso26262/mcdc-per-module suite, db/lanes.json); on every other build * they reduce to a no-op stub so this file still compiles+runs natively. @@ -50,11 +53,87 @@ * the exclusion record + the exclusion record and reports/aes/RESIDUALS.md. */ +/* Class 11 (CPUID_SLOW_VAES) call hooks. + * + * Every VAES (ymm) and AVX-512 (zmm, VAES-gated) AES routine aes.c calls + * directly is renamed to "wb_vaes_hit(), ". A macro is not re-expanded + * inside its own replacement, so + * - in aes.c's declarations it reads "void wb_vaes_hit(), (...) + * XASM_LINK(...)": a second declarator that still declares (and binds) + * the real routine, and + * - at each call site it reads "wb_vaes_hit(), (...);": count, then + * call the real routine. + * So wb_vaes_calls counts every entry into the VAES code without touching the + * link. The declarator form has no prototype, so -Wstrict-prototypes is + * silenced around the include only. The AES-GCM-SIV POLYVAL/CTR routines are + * not hooked: aes.c takes their address, which this form cannot express; + * their selectors are asserted directly instead (wb_slow_vaes_gcmsiv_sel). */ +#if defined(__GNUC__) && defined(__x86_64__) + #define WB_HOOK_VAES + void wb_vaes_hit(void); + #define AES_ECB_encrypt_vaes wb_vaes_hit(), AES_ECB_encrypt_vaes + #define AES_ECB_decrypt_vaes wb_vaes_hit(), AES_ECB_decrypt_vaes + #define AES_CBC_encrypt_vaes wb_vaes_hit(), AES_CBC_encrypt_vaes + #define AES_CBC_decrypt_vaes wb_vaes_hit(), AES_CBC_decrypt_vaes + #define AES_CTR_encrypt_vaes wb_vaes_hit(), AES_CTR_encrypt_vaes + #define AES_GCM_encrypt_vaes wb_vaes_hit(), AES_GCM_encrypt_vaes + #define AES_GCM_decrypt_vaes wb_vaes_hit(), AES_GCM_decrypt_vaes + #define AES_GCM_init_vaes wb_vaes_hit(), AES_GCM_init_vaes + #define AES_GCM_aad_update_vaes wb_vaes_hit(), AES_GCM_aad_update_vaes + #define AES_GCM_encrypt_block_vaes wb_vaes_hit(), AES_GCM_encrypt_block_vaes + #define AES_GCM_ghash_block_vaes wb_vaes_hit(), AES_GCM_ghash_block_vaes + #define AES_GCM_encrypt_update_vaes wb_vaes_hit(), AES_GCM_encrypt_update_vaes + #define AES_GCM_encrypt_final_vaes wb_vaes_hit(), AES_GCM_encrypt_final_vaes + #define AES_GCM_decrypt_update_vaes wb_vaes_hit(), AES_GCM_decrypt_update_vaes + #define AES_GCM_decrypt_final_vaes wb_vaes_hit(), AES_GCM_decrypt_final_vaes + #define AES_XTS_encrypt_vaes wb_vaes_hit(), AES_XTS_encrypt_vaes + #define AES_XTS_decrypt_vaes wb_vaes_hit(), AES_XTS_decrypt_vaes + #define AES_XTS_init_vaes wb_vaes_hit(), AES_XTS_init_vaes + #define AES_XTS_encrypt_update_vaes wb_vaes_hit(), AES_XTS_encrypt_update_vaes + #define AES_XTS_decrypt_update_vaes wb_vaes_hit(), AES_XTS_decrypt_update_vaes + #define AES_ECB_encrypt_avx512 wb_vaes_hit(), AES_ECB_encrypt_avx512 + #define AES_ECB_decrypt_avx512 wb_vaes_hit(), AES_ECB_decrypt_avx512 + #define AES_CBC_encrypt_avx512 wb_vaes_hit(), AES_CBC_encrypt_avx512 + #define AES_CBC_decrypt_avx512 wb_vaes_hit(), AES_CBC_decrypt_avx512 + #define AES_CTR_encrypt_avx512 wb_vaes_hit(), AES_CTR_encrypt_avx512 + #define AES_GCM_encrypt_avx512 wb_vaes_hit(), AES_GCM_encrypt_avx512 + #define AES_GCM_decrypt_avx512 wb_vaes_hit(), AES_GCM_decrypt_avx512 + #define AES_GCM_init_avx512 wb_vaes_hit(), AES_GCM_init_avx512 + #define AES_GCM_aad_update_avx512 wb_vaes_hit(), AES_GCM_aad_update_avx512 + #define AES_GCM_encrypt_block_avx512 wb_vaes_hit(), AES_GCM_encrypt_block_avx512 + #define AES_GCM_ghash_block_avx512 wb_vaes_hit(), AES_GCM_ghash_block_avx512 + #define AES_GCM_encrypt_update_avx512 wb_vaes_hit(), AES_GCM_encrypt_update_avx512 + #define AES_GCM_encrypt_final_avx512 wb_vaes_hit(), AES_GCM_encrypt_final_avx512 + #define AES_GCM_decrypt_update_avx512 wb_vaes_hit(), AES_GCM_decrypt_update_avx512 + #define AES_GCM_decrypt_final_avx512 wb_vaes_hit(), AES_GCM_decrypt_final_avx512 + #define AES_XTS_encrypt_avx512 wb_vaes_hit(), AES_XTS_encrypt_avx512 + #define AES_XTS_decrypt_avx512 wb_vaes_hit(), AES_XTS_decrypt_avx512 + #define AES_XTS_init_avx512 wb_vaes_hit(), AES_XTS_init_avx512 + #define AES_XTS_encrypt_update_avx512 wb_vaes_hit(), AES_XTS_encrypt_update_avx512 + #define AES_XTS_decrypt_update_avx512 wb_vaes_hit(), AES_XTS_decrypt_update_avx512 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstrict-prototypes" + #if defined(__clang__) && defined(__has_warning) + #if __has_warning("-Wdeprecated-non-prototype") + #pragma GCC diagnostic ignored "-Wdeprecated-non-prototype" + #endif + #endif +#endif + /* Pull aes.c in verbatim so the file-static and WOLFSSL_LOCAL helpers below are * in scope and instrumented in THIS binary. aes.c includes settings.h (which * picks up user_settings.h via -DWOLFSSL_USER_SETTINGS) and aes.h itself. */ #include +#ifdef WB_HOOK_VAES + #pragma GCC diagnostic pop + static unsigned long wb_vaes_calls = 0; + void wb_vaes_hit(void) + { + wb_vaes_calls++; + } +#endif + #include #ifndef INVALID_DEVID @@ -63,6 +142,11 @@ static int wb_fail = 0; #define WB_NOTE(msg) do { printf(" [wb] %s\n", (msg)); } while (0) +/* A wrong result, as opposed to a skipped setup: reported as "with failures", + * which run-whitebox-smoke.sh fails on. */ +static int wb_err = 0; +#define WB_CHECK(cond, msg) do { if (!(cond)) { \ + printf(" [wb] FAIL: %s\n", (msg)); wb_err = 1; } } while (0) /* ------------------------------------------------------------------------- * * Class 1a: classic GHASH() internal guards. @@ -1174,6 +1258,356 @@ static void wb_aarch64_ctr_leftover(void) { WB_NOTE("aarch64 base CTR body not compiled in this variant; skipped"); } #endif +/* ------------------------------------------------------------------------- * + * Class 11: CPUID_SLOW_VAES fallback (x86_64, WOLFSSL_AESNI + VAES built). + * + * #define AES_USE_VAES(f) (IS_INTEL_VAES(f) && !IS_CPU_SLOW_VAES(f)) + * + * CPUID_SLOW_VAES is only set from cpuid on Intel Arrow Lake / Lunar Lake, so + * on any other VAES host the fallback never runs. Here intel_flags is forced + * per row, keeping the host's real instruction-feature bits (so only code + * this CPU can execute is reached): + * row 0 host widest VAES arm (zmm on AVX-512 hosts) + * row 1 host - AVX512F ymm VAES arm + * row 2 host + SLOW_VAES fallback (reference) + * row 3 host - AVX512F + SLOW_VAES + * For each row ECB, CBC, CTR, GCM (one-shot and streaming), XTS (one-shot + * and streaming) and AES-GCM-SIV are run at 128/192/256-bit keys on inputs + * past every VAES threshold, decrypted and tag-checked; every row's + * ciphertexts and tags must equal row 2's. The hooks above count entries + * into VAES code: a SLOW_VAES row must make none, and on a VAES host a row + * without it must make some (so the check is not vacuous). + * ------------------------------------------------------------------------- */ +#if defined(WB_HOOK_VAES) && defined(WOLFSSL_AESNI) && \ + defined(HAVE_INTEL_VAES) && defined(USE_INTEL_SPEEDUP) + +#define WB_SV_MSG (64 * WC_AES_BLOCK_SIZE + 5) /* 1029: tail on stream */ +#define WB_SV_BLK (64 * WC_AES_BLOCK_SIZE) /* 1024: ECB/CBC */ +#define WB_SV_AAD 37 +#define WB_SV_OUTCAP (48 * 1024) + +typedef struct WbSink { + byte* p; + word32 n; +} WbSink; + +static byte wb_sv_ref[WB_SV_OUTCAP]; +static byte wb_sv_cur[WB_SV_OUTCAP]; + +static void wb_sv_put(WbSink* s, const byte* d, word32 n) +{ + if (s->n + n > WB_SV_OUTCAP) { + WB_CHECK(0, "SLOW_VAES: output sink overflow"); + return; + } + XMEMCPY(s->p + s->n, d, n); + s->n += n; +} + +static void wb_sv_fill(byte* b, word32 n, byte seed) +{ + word32 i; + for (i = 0; i < n; i++) { + b[i] = (byte)(seed + i * 7u + (i >> 8)); + } +} + +/* Run every AES mode once at key length keySz, appending ciphertexts and + * tags to s and checking each decrypt/verify. */ +static void wb_sv_modes(WbSink* s, word32 keySz) +{ + byte key[64], iv[WC_AES_BLOCK_SIZE], aad[WB_SV_AAD]; + byte pt[WB_SV_MSG], ct[WB_SV_MSG], rt[WB_SV_MSG]; + byte tag[WC_AES_BLOCK_SIZE]; + Aes aes; + int ret; + (void)tag; (void)aad; (void)rt; + + wb_sv_fill(key, sizeof(key), (byte)keySz); + wb_sv_fill(iv, sizeof(iv), 0x5a); + wb_sv_fill(aad, sizeof(aad), 0x33); + wb_sv_fill(pt, sizeof(pt), 0x11); + +#ifdef HAVE_AES_ECB + ret = wc_AesInit(&aes, NULL, INVALID_DEVID); + if (ret == 0) ret = wc_AesSetKey(&aes, key, keySz, NULL, AES_ENCRYPTION); + if (ret == 0) ret = wc_AesEcbEncrypt(&aes, ct, pt, WB_SV_BLK); + WB_CHECK(ret == 0, "SLOW_VAES: ECB encrypt"); + wb_sv_put(s, ct, WB_SV_BLK); + #ifdef HAVE_AES_DECRYPT + if (ret == 0) ret = wc_AesSetKey(&aes, key, keySz, NULL, AES_DECRYPTION); + if (ret == 0) ret = wc_AesEcbDecrypt(&aes, rt, ct, WB_SV_BLK); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_BLK) == 0, + "SLOW_VAES: ECB decrypt"); + #endif + wc_AesFree(&aes); +#endif + +#ifdef HAVE_AES_CBC + ret = wc_AesInit(&aes, NULL, INVALID_DEVID); + if (ret == 0) ret = wc_AesSetKey(&aes, key, keySz, iv, AES_ENCRYPTION); + if (ret == 0) ret = wc_AesCbcEncrypt(&aes, ct, pt, WB_SV_BLK); + WB_CHECK(ret == 0, "SLOW_VAES: CBC encrypt"); + wb_sv_put(s, ct, WB_SV_BLK); + #ifdef HAVE_AES_DECRYPT + if (ret == 0) ret = wc_AesSetKey(&aes, key, keySz, iv, AES_DECRYPTION); + if (ret == 0) ret = wc_AesCbcDecrypt(&aes, rt, ct, WB_SV_BLK); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_BLK) == 0, + "SLOW_VAES: CBC decrypt"); + #endif + wc_AesFree(&aes); +#endif + +#ifdef WOLFSSL_AES_COUNTER + ret = wc_AesInit(&aes, NULL, INVALID_DEVID); + if (ret == 0) ret = wc_AesSetKeyDirect(&aes, key, keySz, iv, + AES_ENCRYPTION); + if (ret == 0) ret = wc_AesCtrEncrypt(&aes, ct, pt, WB_SV_MSG); + WB_CHECK(ret == 0, "SLOW_VAES: CTR encrypt"); + wb_sv_put(s, ct, WB_SV_MSG); + if (ret == 0) ret = wc_AesSetKeyDirect(&aes, key, keySz, iv, + AES_ENCRYPTION); + if (ret == 0) ret = wc_AesCtrEncrypt(&aes, rt, ct, WB_SV_MSG); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: CTR decrypt"); + wc_AesFree(&aes); +#endif + +#ifdef HAVE_AESGCM + ret = wc_AesInit(&aes, NULL, INVALID_DEVID); + if (ret == 0) ret = wc_AesGcmSetKey(&aes, key, keySz); + if (ret == 0) ret = wc_AesGcmEncrypt(&aes, ct, pt, WB_SV_MSG, iv, + GCM_NONCE_MID_SZ, tag, sizeof(tag), + aad, sizeof(aad)); + WB_CHECK(ret == 0, "SLOW_VAES: GCM encrypt"); + wb_sv_put(s, ct, WB_SV_MSG); + wb_sv_put(s, tag, sizeof(tag)); + #ifdef HAVE_AES_DECRYPT + if (ret == 0) ret = wc_AesGcmDecrypt(&aes, rt, ct, WB_SV_MSG, iv, + GCM_NONCE_MID_SZ, tag, sizeof(tag), + aad, sizeof(aad)); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: GCM decrypt/verify"); + #endif + wc_AesFree(&aes); + + #ifdef WOLFSSL_AESGCM_STREAM + /* AAD first, then the message in two uneven pieces so both the bulk + * update and the partial-block carry run. */ + ret = wc_AesInit(&aes, NULL, INVALID_DEVID); + if (ret == 0) ret = wc_AesGcmInit(&aes, key, keySz, iv, GCM_NONCE_MID_SZ); + if (ret == 0) ret = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0, aad, + sizeof(aad)); + if (ret == 0) ret = wc_AesGcmEncryptUpdate(&aes, ct, pt, 1000, NULL, 0); + if (ret == 0) ret = wc_AesGcmEncryptUpdate(&aes, ct + 1000, pt + 1000, + WB_SV_MSG - 1000, NULL, 0); + if (ret == 0) ret = wc_AesGcmEncryptFinal(&aes, tag, sizeof(tag)); + WB_CHECK(ret == 0, "SLOW_VAES: GCM stream encrypt"); + wb_sv_put(s, ct, WB_SV_MSG); + wb_sv_put(s, tag, sizeof(tag)); + #ifdef HAVE_AES_DECRYPT + if (ret == 0) ret = wc_AesGcmInit(&aes, key, keySz, iv, GCM_NONCE_MID_SZ); + if (ret == 0) ret = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0, aad, + sizeof(aad)); + if (ret == 0) ret = wc_AesGcmDecryptUpdate(&aes, rt, ct, 1000, NULL, 0); + if (ret == 0) ret = wc_AesGcmDecryptUpdate(&aes, rt + 1000, ct + 1000, + WB_SV_MSG - 1000, NULL, 0); + if (ret == 0) ret = wc_AesGcmDecryptFinal(&aes, tag, sizeof(tag)); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: GCM stream decrypt/verify"); + #endif + wc_AesFree(&aes); + #endif +#endif + +#ifdef WOLFSSL_AESGCM_SIV + /* RFC 8452 defines AES-GCM-SIV for 128- and 256-bit keys only. */ + if (keySz != 24) { + ret = wc_AesGcmSivEncrypt(key, keySz, iv, GCM_NONCE_MID_SZ, aad, + sizeof(aad), pt, WB_SV_MSG, ct, tag, + sizeof(tag)); + WB_CHECK(ret == 0, "SLOW_VAES: GCM-SIV encrypt"); + wb_sv_put(s, ct, WB_SV_MSG); + wb_sv_put(s, tag, sizeof(tag)); + if (ret == 0) ret = wc_AesGcmSivDecrypt(key, keySz, iv, GCM_NONCE_MID_SZ, + aad, sizeof(aad), ct, WB_SV_MSG, + rt, tag, sizeof(tag)); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: GCM-SIV decrypt/verify"); + } +#endif + +#if defined(WOLFSSL_AES_XTS) && defined(HAVE_AES_DECRYPT) + /* XTS takes a double-length key; 192-bit is not an XTS size. */ + if (keySz != 24) { + XtsAes xts; + word32 xKeySz = keySz * 2; + + ret = wc_AesXtsSetKey(&xts, key, xKeySz, AES_ENCRYPTION, NULL, + INVALID_DEVID); + if (ret == 0) ret = wc_AesXtsEncrypt(&xts, ct, pt, WB_SV_MSG, iv, + sizeof(iv)); + WB_CHECK(ret == 0, "SLOW_VAES: XTS encrypt"); + wb_sv_put(s, ct, WB_SV_MSG); + #ifdef WOLFSSL_AESXTS_STREAM + { + struct XtsAesStreamData stream; + byte sct[WB_SV_MSG]; + /* 63 whole blocks through Update, the last block plus the + * 5-byte tail (ciphertext stealing) through Final. */ + word32 head = 63 * WC_AES_BLOCK_SIZE; + + if (ret == 0) ret = wc_AesXtsEncryptInit(&xts, iv, sizeof(iv), + &stream); + if (ret == 0) ret = wc_AesXtsEncryptUpdate(&xts, sct, pt, head, + &stream); + if (ret == 0) ret = wc_AesXtsEncryptFinal(&xts, sct + head, + pt + head, + WB_SV_MSG - head, + &stream); + WB_CHECK(ret == 0 && XMEMCMP(sct, ct, WB_SV_MSG) == 0, + "SLOW_VAES: XTS stream encrypt != one-shot"); + } + #endif + wc_AesXtsFree(&xts); + + ret = wc_AesXtsSetKey(&xts, key, xKeySz, AES_DECRYPTION, NULL, + INVALID_DEVID); + if (ret == 0) ret = wc_AesXtsDecrypt(&xts, rt, ct, WB_SV_MSG, iv, + sizeof(iv)); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: XTS decrypt"); + #ifdef WOLFSSL_AESXTS_STREAM + { + struct XtsAesStreamData stream; + word32 head = 63 * WC_AES_BLOCK_SIZE; + + XMEMSET(rt, 0, sizeof(rt)); + if (ret == 0) ret = wc_AesXtsDecryptInit(&xts, iv, sizeof(iv), + &stream); + if (ret == 0) ret = wc_AesXtsDecryptUpdate(&xts, rt, ct, head, + &stream); + if (ret == 0) ret = wc_AesXtsDecryptFinal(&xts, rt + head, + ct + head, + WB_SV_MSG - head, + &stream); + WB_CHECK(ret == 0 && XMEMCMP(rt, pt, WB_SV_MSG) == 0, + "SLOW_VAES: XTS stream decrypt"); + } + #endif + wc_AesXtsFree(&xts); + } +#endif +} + +/* The GCM-SIV POLYVAL and CTR routines are reached through a returned + * function pointer, so assert the selection itself. */ +static void wb_slow_vaes_gcmsiv_sel(int slow, int hostVaes) +{ +#if defined(WOLFSSL_AESGCM_SIV) && defined(WC_POLYVAL_ASM) && \ + defined(WC_GCMSIV_CTR_ASM) && defined(WOLFSSL_X86_64_BUILD) && \ + !defined(WC_POLYVAL_ASM_AARCH64) && !defined(WC_POLYVAL_ASM_AARCH32) + AesGcmSivPolyvalFn pfn = AesGcmSivPolyvalAsm(); + AesGcmSivCtrFn cfn = AesGcmSivCtrAsm(); + int pVaes = (pfn == &AES_GCMSIV_polyval_vaes); + int cVaes = (cfn == &AES_GCMSIV_ctr_vaes); + #ifdef HAVE_INTEL_AVX512 + pVaes |= (pfn == &AES_GCMSIV_polyval_avx512); + cVaes |= (cfn == &AES_GCMSIV_ctr_avx512); + #endif + if (slow) { + WB_CHECK(!pVaes, "SLOW_VAES: GCM-SIV POLYVAL still selects VAES"); + WB_CHECK(!cVaes, "SLOW_VAES: GCM-SIV CTR still selects VAES"); + } + else if (hostVaes) { + WB_CHECK(pVaes, "SLOW_VAES clear: GCM-SIV POLYVAL did not pick VAES"); + WB_CHECK(cVaes, "SLOW_VAES clear: GCM-SIV CTR did not pick VAES"); + } +#else + (void)slow; (void)hostVaes; +#endif +} + +static void wb_slow_vaes(void) +{ + static const word32 keySzs[3] = { 16, 24, 32 }; + static const int order[4] = { 2, 0, 1, 3 }; + cpuid_flags_t saved, base, rows[4]; + WbSink cur; + word32 refLen = 0; + int hostVaes, i; + unsigned k; + + /* AES_USE_VAES truth table on synthetic flags (decision only, nothing + * executes): both operands' independence pairs. */ + WB_CHECK(AES_USE_VAES((cpuid_flags_t)CPUID_VAES) != 0, + "AES_USE_VAES(VAES) is false"); + WB_CHECK(AES_USE_VAES((cpuid_flags_t)(CPUID_VAES | CPUID_SLOW_VAES)) == 0, + "AES_USE_VAES(VAES|SLOW_VAES) is true"); + WB_CHECK(AES_USE_VAES((cpuid_flags_t)0) == 0, + "AES_USE_VAES(0) is true"); + + cpuid_get_flags_ex(&intel_flags); + saved = intel_flags; + if (!IS_INTEL_AESNI(saved)) { + WB_NOTE("host has no AES-NI; SLOW_VAES rows skipped"); + return; + } + /* The host's own tuning bit is cleared so row 0 means the same on + * Arrow Lake as elsewhere. */ + base = saved & ~(cpuid_flags_t)CPUID_SLOW_VAES; + hostVaes = IS_INTEL_VAES(base) && + (IS_INTEL_AVX2(base) || IS_INTEL_AVX512(base)); + rows[0] = base; + rows[1] = base & ~(cpuid_flags_t)CPUID_AVX512; + rows[2] = base | CPUID_SLOW_VAES; + rows[3] = rows[1] | CPUID_SLOW_VAES; + + /* Row 2 (the fallback) runs first and is the reference. */ + for (i = 0; i < 4; i++) { + int r = order[i]; + int slow = IS_CPU_SLOW_VAES(rows[r]) != 0; + + cur.p = (r == 2) ? wb_sv_ref : wb_sv_cur; + cur.n = 0; + intel_flags = rows[r]; + wb_vaes_calls = 0; + for (k = 0; k < sizeof(keySzs) / sizeof(keySzs[0]); k++) { + wb_sv_modes(&cur, keySzs[k]); + } + wb_slow_vaes_gcmsiv_sel(slow, hostVaes); + + printf(" [wb] SLOW_VAES row %d (flags 0x%x): %lu VAES calls\n", r, + (unsigned)rows[r], wb_vaes_calls); + if (slow) { + WB_CHECK(wb_vaes_calls == 0, + "SLOW_VAES set but VAES code was called"); + } + else if (hostVaes && (r == 0 || IS_INTEL_AVX2(rows[r]))) { + WB_CHECK(wb_vaes_calls > 0, + "SLOW_VAES clear on a VAES host but no VAES call " + "(hooks or thresholds broken)"); + } + if (r == 2) { + refLen = cur.n; + } + else { + WB_CHECK(cur.n == refLen && + XMEMCMP(wb_sv_cur, wb_sv_ref, refLen) == 0, + "SLOW_VAES: ciphertext/tag differs from fallback"); + } + } + intel_flags = saved; + if (!hostVaes) { + WB_NOTE("host has no VAES: rows compare the fallback with itself"); + } + WB_NOTE("CPUID_SLOW_VAES fallback rows exercised"); +} +#else +static void wb_slow_vaes(void) +{ WB_NOTE("x86_64 VAES AES not built in this variant; SLOW_VAES skipped"); } +#endif + int main(void) { setvbuf(stdout, NULL, _IONBF, 0); @@ -1196,9 +1630,13 @@ int main(void) wb_ccm_roll_auth_ret(); wb_ccm_aesni_dispatch(); wb_aarch64_ctr_leftover(); - printf("done (%s)\n", wb_fail ? "with skips" : "ok"); + wb_slow_vaes(); + printf("done (%s)\n", wb_err ? "with failures" : + wb_fail ? "with skips" : "ok"); /* Setup failures are surfaced as skips, not test failures: the harness - * treats a nonzero exit as a failed variant and discards its coverage. */ + * treats a nonzero exit as a failed variant and discards its coverage. + * Wrong results (wb_err) are reported as "with failures", which + * run-whitebox-smoke.sh fails on, and still exit 0 for the same reason. */ return 0; #endif } diff --git a/tests/unit-mcdc/test_chacha_whitebox.c b/tests/unit-mcdc/test_chacha_whitebox.c index 18ff0d538b9..9d4410d53e6 100644 --- a/tests/unit-mcdc/test_chacha_whitebox.c +++ b/tests/unit-mcdc/test_chacha_whitebox.c @@ -57,20 +57,21 @@ * The AVX-512 ChaCha work replaced that flat if/else with a length-and-feature * ladder in wc_Chacha_Process, five compound decisions deep: * - * msglen <= CHACHA_CHUNK_BYTES && IS_INTEL_AVX512_VL(f) == 0 && + * msglen <= CHACHA_CHUNK_BYTES && !CHACHA_USE_AVX512_VL(f) && * IS_INTEL_AVX1(f) -> 1-block AVX1 - * IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK && + * !CHACHA_USE_AVX512_VL(f) && msglen < 4*CHUNK && * IS_INTEL_SSSE3(f) -> SSSE3 - * IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK -> scalar x64 + * !CHACHA_USE_AVX512_VL(f) && msglen < 4*CHUNK -> scalar x64 * chacha_avx512_beneficial(f) && msglen >= 16*CHUNK -> 512-bit zmm - * IS_INTEL_AVX512_VL(f) && msglen < 8*CHUNK -> AVX-512VL + * CHACHA_USE_AVX512_VL(f) && msglen < 8*CHUNK -> AVX-512VL * * plus the predicate chacha_avx512_beneficial(): IS_INTEL_AVX512(f) && - * IS_INTEL_VAES(f). + * IS_INTEL_VAES(f). CHACHA_USE_AVX512_VL(f) is IS_INTEL_AVX512_VL(f) when the + * AVX-512VL kernel is built (HAVE_INTEL_AVX512) and 0 otherwise. * * Two inputs decide all of them: the message length and the cpuid mask. One * host only ever supplies one mask -- on an AVX-512VL machine every - * "IS_INTEL_AVX512_VL(f) == 0" operand is permanently false, which pins three + * "!CHACHA_USE_AVX512_VL(f)" operand is permanently false, which pins three * of the five decisions to a single outcome no matter what lengths the tests * use, and on a machine without AVX-512 the last two are pinned the other way. * wb_chacha_process_case() drives one (mask, length) pair; the table in diff --git a/tests/unit-mcdc/test_sha3_whitebox.c b/tests/unit-mcdc/test_sha3_whitebox.c index fb5d1ba6890..bc58a6ab015 100644 --- a/tests/unit-mcdc/test_sha3_whitebox.c +++ b/tests/unit-mcdc/test_sha3_whitebox.c @@ -108,13 +108,17 @@ static void wb_sha3_dispatch(void) wb_init_with(CPUID_BMI1, NULL); /* [T,F] */ wb_init_with(0, NULL); /* [F,-] -> C block */ - /* SHA3_USE_AVX2(f) = IS_INTEL_AVX2(f) && IS_CPU_INTEL(f). + /* SHA3_USE_AVX2(f) = IS_INTEL_AVX2(f) && IS_CPU_INTEL(f) && + * !IS_CPU_SLOW_AVX2_SHA3(f). * The [T,T] row cannot come from real cpuid on a non-Intel host, so * neither operand's pair completes without forcing the vendor bit: * AVX2-capable AMD reports [T,F], the same row as the AVX2-only call * above. Supply [T,T]; it pairs with CPUID_AVX2 ([T,F]) for operand 1 * and with the BMI rows ([F,-]) for operand 0. */ wb_init_with(CPUID_AVX2 | CPUID_INTEL, NULL); + /* Third operand, !IS_CPU_SLOW_AVX2_SHA3(f): [T,T,T] pairs with the row + * above for its independence (Arrow Lake: AVX2 Keccak skipped). */ + wb_init_with(CPUID_AVX2 | CPUID_INTEL | CPUID_SLOW_AVX2_SHA3, NULL); /* 874: (sha3_block_n != NULL) && (blocks > 0), in Sha3Update. MC/DC needs * cond0's independence pair -- both the NULL and non-NULL multi-block diff --git a/tests/unit-mcdc/test_wc_mldsa_whitebox.c b/tests/unit-mcdc/test_wc_mldsa_whitebox.c index bf75a41c2ec..fc13a4e9ba4 100644 --- a/tests/unit-mcdc/test_wc_mldsa_whitebox.c +++ b/tests/unit-mcdc/test_wc_mldsa_whitebox.c @@ -1076,9 +1076,12 @@ static void wb_dispatch_rows(void) /* USE_INTEL_AVX512(f) is itself IS_INTEL_AVX512(f) && IS_INTEL_AVX512_BW(f) * (cpuid.h), so each AVX512 dispatch is a three-condition decision and the * F and BW bits need to be cleared separately. SHA3_USE_AVX2(f) is - * IS_INTEL_AVX2(f) && IS_CPU_INTEL(f): its vendor operand is false on any - * AMD host, so CPUID_INTEL is forced on for the rows that need its true - * side -- the arm behind it is plain AVX2, which runs anywhere AVX2 does. */ + * IS_INTEL_AVX2(f) && IS_CPU_INTEL(f) && !IS_CPU_SLOW_AVX2_SHA3(f): its + * vendor operand is false on any AMD host, so CPUID_INTEL is forced on for + * the rows that need its true side -- the arm behind it is plain AVX2, + * which runs anywhere AVX2 does. The host's tuning bits are cleared on + * every row so the rows mean the same on Arrow Lake, and one row sets + * CPUID_SLOW_AVX2_SHA3 for that operand's true side. */ static const struct { cpuid_flags_t set; cpuid_flags_t clear; @@ -1106,6 +1109,9 @@ static void wb_dispatch_rows(void) CPUID_AVX512_VBMI | CPUID_BMI2 | CPUID_AVX2, 0 }, /* portable C */ { 0, CPUID_INTEL, 0 }, /* non-Intel vendor */ + { CPUID_INTEL | CPUID_SLOW_AVX2_SHA3, + CPUID_AVX512 | CPUID_AVX512_BW | + CPUID_AVX512_VBMI, 0 }, /* AVX2, slow Keccak */ }; if (wc_InitRng(&rng) != 0) { @@ -1116,6 +1122,7 @@ static void wb_dispatch_rows(void) for (i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) { cpuid_flags = WC_CPUID_INITIALIZER; (void)cpuid_get_flags_ex(&cpuid_flags); + cpuid_flags &= (cpuid_flags_t)~(CPUID_SLOW_VAES | CPUID_SLOW_AVX2_SHA3); cpuid_flags |= rows[i].set; cpuid_flags &= (cpuid_flags_t)~rows[i].clear; wb_intr_ret = rows[i].intr; diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index f9f8b4c45a4..09df60af092 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -5168,6 +5168,8 @@ static void print_cpu_features(void) if (IS_INTEL_SHA(cpuid_flags)) printf(" sha"); if (IS_INTEL_VAES(cpuid_flags)) printf(" vaes"); if (IS_INTEL_AVX512(cpuid_flags)) printf(" avx512"); + if (IS_CPU_SLOW_VAES(cpuid_flags)) printf(" (slow-vaes)"); + if (IS_CPU_SLOW_AVX2_SHA3(cpuid_flags)) printf(" (slow-avx2-sha3)"); #endif #ifdef __aarch64__ printf("Aarch64 -"); diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 8659e65f12f..b8b458bda4c 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -972,6 +972,13 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) #define HAVE_INTEL_AVX512 #endif + /* VAES is available and worth using for AES on this CPU. CPUs marked + * CPUID_SLOW_VAES (Intel Arrow Lake and Lunar Lake) run the VAES code + * no faster than the AVX2 / AES-NI code, so they use that instead. + * Also used for the GCM-SIV POLYVAL selection. */ + #define AES_USE_VAES(f) \ + (IS_INTEL_VAES(f) && !IS_CPU_SLOW_VAES(f)) + /* Below this threshold the narrower path (AVX1 / AES-NI) is faster on * Zen 4 than the wide VAES/AVX512 path. Verify and tune * per-microarchecture. @@ -1058,14 +1065,14 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_ECB_encrypt_avx512(in, out, sz, key, nr); } else #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_ECB_encrypt_vaes(in, out, sz, key, nr); } else @@ -1088,14 +1095,14 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_ECB_decrypt_avx512(in, out, sz, key, nr); } else #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_ECB_decrypt_vaes(in, out, sz, key, nr); } else @@ -1119,14 +1126,14 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CBC_encrypt_avx512(in, out, iv, sz, key, nr); } else #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CBC_encrypt_vaes(in, out, iv, sz, key, nr); } else @@ -1150,14 +1157,14 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CBC_decrypt_avx512(in, out, iv, sz, key, nr); } else #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CBC_decrypt_vaes(in, out, iv, sz, key, nr); } else @@ -1181,14 +1188,14 @@ static WC_INLINE void wc_Stm32_CrypAesBlock(const byte* in, byte* out) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CTR_encrypt_avx512(in, out, sz, key, nr, ctr); } else #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_ECB_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_CTR_encrypt_vaes(in, out, sz, key, nr, ctr); } else @@ -12355,7 +12362,7 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_avx512(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (const byte*)aes->key, (int)aes->rounds); ret = 0; @@ -12364,7 +12371,7 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_vaes(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (const byte*)aes->key, (int)aes->rounds); ret = 0; @@ -13271,7 +13278,7 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { #ifdef HAVE_INTEL_AVX512 if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_avx512(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (byte*)aes->key, (int)aes->rounds, &res); if (res == 0) @@ -13283,7 +13290,7 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, #endif #ifdef HAVE_INTEL_VAES if ((sz >= WC_AES_BLOCK_SIZE * WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_vaes(in, out, authIn, iv, authTag, sz, authInSz, ivSz, authTagSz, (byte*)aes->key, (int)aes->rounds, &res); if (res == 0) @@ -13674,14 +13681,14 @@ static WARN_UNUSED_RESULT int AesGcmInit_asm( aes->cOver = 0; #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_init_avx512((byte*)aes->key, (int)aes->rounds, iv, ivSz, aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes)); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_init_vaes((byte*)aes->key, (int)aes->rounds, iv, ivSz, aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes)); } @@ -13742,14 +13749,14 @@ static WARN_UNUSED_RESULT int AesGcmAadUpdate_asm( if (aes->aOver == WC_AES_BLOCK_SIZE) { /* We have filled up the block and can process. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } @@ -13788,7 +13795,7 @@ static WARN_UNUSED_RESULT int AesGcmAadUpdate_asm( /* GHASH full blocks now. */ #ifdef HAVE_INTEL_AVX512 if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_aad_update_avx512(a, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H); } @@ -13796,7 +13803,7 @@ static WARN_UNUSED_RESULT int AesGcmAadUpdate_asm( #endif #ifdef HAVE_INTEL_VAES if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_aad_update_vaes(a, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H); } @@ -13836,14 +13843,14 @@ static WARN_UNUSED_RESULT int AesGcmAadUpdate_asm( (size_t)WC_AES_BLOCK_SIZE - aes->aOver); /* GHASH last AAD block. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } @@ -13917,14 +13924,14 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_asm( if (aes->cOver == WC_AES_BLOCK_SIZE) { /* We have filled up the block and can process. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } @@ -13964,7 +13971,7 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_asm( /* Encrypt and GHASH full blocks now. */ #ifdef HAVE_INTEL_AVX512 if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_update_avx512((byte*)aes->key, (int)aes->rounds, c, p, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes)); @@ -13973,7 +13980,7 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_asm( #endif #ifdef HAVE_INTEL_VAES if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_update_vaes((byte*)aes->key, (int)aes->rounds, c, p, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes)); @@ -14009,14 +14016,14 @@ static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_asm( /* Encrypt the counter - XOR in zeros as proxy for plaintext. */ XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE); #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_block_avx512((byte*)aes->key, (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes)); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_block_vaes((byte*)aes->key, (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes)); } @@ -14078,14 +14085,14 @@ static WARN_UNUSED_RESULT int AesGcmEncryptFinal_asm( XMEMSET(AES_LASTGBLOCK(aes) + over, 0, (size_t)WC_AES_BLOCK_SIZE - over); /* GHASH last cipher block. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(AES_LASTGBLOCK(aes), AES_TAG(aes), aes->gcm.H); } @@ -14112,14 +14119,14 @@ static WARN_UNUSED_RESULT int AesGcmEncryptFinal_asm( } /* Calculate the authentication tag. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_final_avx512(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes)); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_final_vaes(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes)); } @@ -14244,14 +14251,14 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_asm( if (aes->cOver == WC_AES_BLOCK_SIZE) { /* We have filled up the block and can process. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(AES_LASTBLOCK(aes), AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(AES_LASTBLOCK(aes), AES_TAG(aes), aes->gcm.H); } @@ -14291,7 +14298,7 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_asm( /* Decrypt and GHASH full blocks now. */ #ifdef HAVE_INTEL_AVX512 if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_update_avx512((byte*)aes->key, (int)aes->rounds, p, c, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes)); @@ -14300,7 +14307,7 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_asm( #endif #ifdef HAVE_INTEL_VAES if ((blocks >= WC_VAES_GCM_MIN_BLOCKS) && - IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_update_vaes((byte*)aes->key, (int)aes->rounds, p, c, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes)); @@ -14336,14 +14343,14 @@ static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_asm( /* Encrypt the counter - XOR in zeros as proxy for cipher text. */ XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE); #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_block_avx512((byte*)aes->key, (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes)); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_encrypt_block_vaes((byte*)aes->key, (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes)); } @@ -14412,13 +14419,13 @@ static WARN_UNUSED_RESULT int AesGcmDecryptFinal_asm( XMEMSET(lastBlock + over, 0, (size_t)WC_AES_BLOCK_SIZE - over); /* Hash the last block of cipher text. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_avx512(lastBlock, AES_TAG(aes), aes->gcm.H); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_ghash_block_vaes(lastBlock, AES_TAG(aes), aes->gcm.H); } else @@ -14441,14 +14448,14 @@ static WARN_UNUSED_RESULT int AesGcmDecryptFinal_asm( } /* Calculate and compare the authentication tag. */ #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_final_avx512(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res); } else #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_GCM_decrypt_final_vaes(AES_TAG(aes), authTag, authTagSz, aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res); } @@ -19863,7 +19870,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_encrypt_avx512(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -19873,7 +19880,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_encrypt_vaes(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -19990,7 +19997,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_init_avx512(stream->tweak_block, (const byte*)xaes->tweak.key, (int)xaes->tweak.rounds); @@ -19999,7 +20006,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_init_vaes(stream->tweak_block, (const byte*)xaes->tweak.key, (int)xaes->tweak.rounds); @@ -20113,7 +20120,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_encrypt_update_avx512(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -20123,7 +20130,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_encrypt_update_vaes(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -20495,7 +20502,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_decrypt_avx512(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -20505,7 +20512,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_decrypt_vaes(in, out, sz, i, (const byte*)aes->key, (const byte*)xaes->tweak.key, @@ -20625,7 +20632,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_init_avx512(stream->tweak_block, (const byte*)xaes->tweak.key, (int)xaes->tweak.rounds); @@ -20634,7 +20641,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_init_vaes(stream->tweak_block, (const byte*)xaes->tweak.key, (int)xaes->tweak.rounds); @@ -20752,7 +20759,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_decrypt_update_avx512(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -20762,7 +20769,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s else #endif #if defined(HAVE_INTEL_VAES) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX2(intel_flags) && AES_USE_VAES(intel_flags)) { AES_XTS_decrypt_update_vaes(in, out, sz, (const byte*)aes->key, stream->tweak_block, @@ -21984,12 +21991,12 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void) return NULL; } #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { return &AES_GCMSIV_polyval_avx512; } #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_VAES(intel_flags) && IS_INTEL_AVX2(intel_flags)) { + if (AES_USE_VAES(intel_flags) && IS_INTEL_AVX2(intel_flags)) { return &AES_GCMSIV_polyval_vaes; } #endif @@ -22056,12 +22063,12 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void) return NULL; } #ifdef HAVE_INTEL_AVX512 - if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) { + if (IS_INTEL_AVX512(intel_flags) && AES_USE_VAES(intel_flags)) { return &AES_GCMSIV_ctr_avx512; } #endif #ifdef HAVE_INTEL_VAES - if (IS_INTEL_VAES(intel_flags) && IS_INTEL_AVX2(intel_flags)) { + if (AES_USE_VAES(intel_flags) && IS_INTEL_AVX2(intel_flags)) { return &AES_GCMSIV_ctr_vaes; } #endif diff --git a/wolfcrypt/src/aes_gcm_asm.S b/wolfcrypt/src/aes_gcm_asm.S index f215edef841..cc89094769b 100644 --- a/wolfcrypt/src/aes_gcm_asm.S +++ b/wolfcrypt/src/aes_gcm_asm.S @@ -470,6 +470,7 @@ _AES_GCM_encrypt_aesni: pushq %r15 movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 48(%rsp), %r11d movl 56(%rsp), %ebx movl 64(%rsp), %r14d @@ -2327,6 +2328,7 @@ _AES_GCM_decrypt_aesni: pushq %rbp movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 56(%rsp), %r11d movl 64(%rsp), %ebx movl 72(%rsp), %r14d @@ -3730,6 +3732,7 @@ _AES_GCM_init_aesni: pushq %r12 pushq %r13 pushq %r14 + movl %esi, %esi movq %rdx, %r10 movl %ecx, %r11d movq 32(%rsp), %rax @@ -4082,6 +4085,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_aad_update_aesni) .p2align 4 _AES_GCM_aad_update_aesni: #endif /* __APPLE__ */ + movl %esi, %esi movq %rcx, %rax movdqa (%rdx), %xmm5 movdqa (%rax), %xmm6 @@ -4166,6 +4170,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_aesni) .p2align 4 _AES_GCM_encrypt_block_aesni: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r10 movq %rcx, %r11 movdqu (%r8), %xmm0 @@ -4300,7 +4305,9 @@ _AES_GCM_encrypt_update_aesni: pushq %r12 pushq %r14 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 32(%rsp), %rax movq 40(%rsp), %r12 subq $0xa0, %rsp @@ -5628,7 +5635,9 @@ _AES_GCM_decrypt_update_aesni: pushq %r14 pushq %r15 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 40(%rsp), %rax movq 48(%rsp), %r12 subq $0xa8, %rsp @@ -6526,6 +6535,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_polyval_aesni) .p2align 4 _AES_GCMSIV_polyval_aesni: #endif /* __APPLE__ */ + movl %ecx, %ecx movdqu (%rsi), %xmm1 movdqu (%rdi), %xmm0 pshufb L_aes_gcm_siv_bswap_mask(%rip), %xmm0 @@ -6891,6 +6901,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_ctr_aesni) _AES_GCMSIV_ctr_aesni: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d movdqu L_aes_gcmsiv_ctr_aesni_one(%rip), %xmm8 movdqu (%r9), %xmm7 xorl %eax, %eax @@ -7455,6 +7466,7 @@ _AES_GCM_encrypt_avx1: pushq %r15 movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 48(%rsp), %r11d movl 56(%rsp), %ebx movl 64(%rsp), %r14d @@ -9040,6 +9052,7 @@ _AES_GCM_decrypt_avx1: pushq %rbp movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 56(%rsp), %r11d movl 64(%rsp), %ebx movl 72(%rsp), %r14d @@ -10213,6 +10226,7 @@ _AES_GCM_init_avx1: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %esi, %esi movq %rdx, %r10 movl %ecx, %r11d movq 24(%rsp), %rax @@ -10530,6 +10544,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_aad_update_avx1) .p2align 4 _AES_GCM_aad_update_avx1: #endif /* __APPLE__ */ + movl %esi, %esi movq %rcx, %rax vmovdqa (%rdx), %xmm5 vmovdqa (%rax), %xmm6 @@ -10603,6 +10618,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_avx1) .p2align 4 _AES_GCM_encrypt_block_avx1: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r10 movq %rcx, %r11 vmovdqu (%r8), %xmm1 @@ -10727,7 +10743,9 @@ _AES_GCM_encrypt_update_avx1: pushq %r12 pushq %r14 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 32(%rsp), %rax movq 40(%rsp), %r12 subq $0xa0, %rsp @@ -11846,7 +11864,9 @@ _AES_GCM_decrypt_update_avx1: pushq %r12 pushq %r14 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 32(%rsp), %rax movq 40(%rsp), %r12 subq $0xb0, %rsp @@ -12578,6 +12598,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_polyval_avx1) .p2align 4 _AES_GCMSIV_polyval_avx1: #endif /* __APPLE__ */ + movl %ecx, %ecx vmovdqu (%rsi), %xmm1 vmovdqu (%rdi), %xmm0 vpshufb L_aes_gcm_siv_bswap_mask_avx1(%rip), %xmm0, %xmm0 @@ -12847,6 +12868,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_ctr_avx1) _AES_GCMSIV_ctr_avx1: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vmovdqu L_aes_gcmsiv_ctr_avx1_one(%rip), %xmm8 vmovdqu (%r9), %xmm7 xorl %eax, %eax @@ -16052,6 +16074,7 @@ _AES_GCM_init_avx2: #endif /* __APPLE__ */ pushq %rbx pushq %r12 + movl %esi, %esi movq %rdx, %r10 movl %ecx, %r11d movq 24(%rsp), %rax @@ -16327,6 +16350,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_aad_update_avx2) .p2align 4 _AES_GCM_aad_update_avx2: #endif /* __APPLE__ */ + movl %esi, %esi movq %rcx, %rax vmovdqu (%rdx), %xmm4 vmovdqu (%rax), %xmm5 @@ -16388,6 +16412,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_avx2) .p2align 4 _AES_GCM_encrypt_block_avx2: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r10 movq %rcx, %r11 subq $0x98, %rsp @@ -16514,7 +16539,9 @@ _AES_GCM_encrypt_update_avx2: pushq %r13 pushq %r14 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 32(%rsp), %rax movq 40(%rsp), %r12 subq $0xa0, %rsp @@ -17370,6 +17397,9 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_final_avx2) .p2align 4 _AES_GCM_encrypt_final_avx2: #endif /* __APPLE__ */ + movl %edx, %edx + movl %ecx, %ecx + movl %r8d, %r8d movq 8(%rsp), %rax subq $24, %rsp vmovdqu (%rdi), %xmm4 @@ -17448,7 +17478,9 @@ _AES_GCM_decrypt_update_avx2: pushq %r12 pushq %r14 movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 32(%rsp), %rax movq 40(%rsp), %r12 subq $0xb0, %rsp @@ -17958,6 +17990,9 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_decrypt_final_avx2) _AES_GCM_decrypt_final_avx2: #endif /* __APPLE__ */ pushq %r12 + movl %edx, %edx + movl %ecx, %ecx + movl %r8d, %r8d movq 16(%rsp), %rax movq 24(%rsp), %r10 subq $16, %rsp @@ -18473,6 +18508,7 @@ _AES_GCM_encrypt_vaes: pushq %r15 movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 48(%rsp), %r11d movl 56(%rsp), %ebx movl 64(%rsp), %r14d @@ -20479,6 +20515,7 @@ _AES_GCM_decrypt_vaes: pushq %rbp movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 56(%rsp), %r11d movl 64(%rsp), %ebx movl 72(%rsp), %r14d @@ -22434,6 +22471,7 @@ _AES_GCM_init_vaes: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %esi, %esi movq %rdx, %r10 movl %ecx, %r11d movq 24(%rsp), %rax @@ -22751,6 +22789,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_aad_update_vaes) .p2align 4 _AES_GCM_aad_update_vaes: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r8 movq %rcx, %r9 subq $0x208, %rsp @@ -23286,6 +23325,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_vaes) .p2align 4 _AES_GCM_encrypt_block_vaes: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r10 movq %rcx, %r11 vmovdqu (%r8), %xmm1 @@ -23412,7 +23452,9 @@ _AES_GCM_encrypt_update_vaes: pushq %r15 pushq %rbx movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 48(%rsp), %rax movq 56(%rsp), %r12 subq $0x210, %rsp @@ -24447,7 +24489,9 @@ _AES_GCM_decrypt_update_vaes: pushq %r15 pushq %rbx movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 48(%rsp), %rax movq 56(%rsp), %r12 subq $0x210, %rsp @@ -25458,6 +25502,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_polyval_vaes) .p2align 4 _AES_GCMSIV_polyval_vaes: #endif /* __APPLE__ */ + movl %ecx, %ecx vmovdqu (%rsi), %xmm1 vmovdqu (%rdi), %xmm0 vpshufb L_aes_gcm_siv_bswap_mask_vaes(%rip), %xmm0, %xmm0 @@ -26007,6 +26052,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_ctr_vaes) _AES_GCMSIV_ctr_vaes: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vbroadcasti128 (%r9), %ymm7 vbroadcasti128 128+L_aes_gcmsiv_ctr_inc_vaes(%rip), %ymm8 vbroadcasti128 32+L_aes_gcmsiv_ctr_inc_vaes(%rip), %ymm9 @@ -26698,6 +26744,7 @@ _AES_GCM_encrypt_avx512: pushq %r15 movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 48(%rsp), %r11d movl 56(%rsp), %ebx movl 64(%rsp), %r14d @@ -29403,6 +29450,7 @@ _AES_GCM_decrypt_avx512: pushq %rbp movq %rdx, %r12 movq %rcx, %rax + movl %r9d, %r9d movl 56(%rsp), %r11d movl 64(%rsp), %ebx movl 72(%rsp), %r14d @@ -31902,6 +31950,7 @@ _AES_GCM_init_avx512: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %esi, %esi movq %rdx, %r10 movl %ecx, %r11d movq 24(%rsp), %rax @@ -32219,6 +32268,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_aad_update_avx512) .p2align 4 _AES_GCM_aad_update_avx512: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r8 movq %rcx, %r9 subq $0x108, %rsp @@ -32653,6 +32703,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_avx512) .p2align 4 _AES_GCM_encrypt_block_avx512: #endif /* __APPLE__ */ + movl %esi, %esi movq %rdx, %r10 movq %rcx, %r11 vmovdqu (%r8), %xmm1 @@ -32780,7 +32831,9 @@ _AES_GCM_encrypt_update_avx512: pushq %rbx pushq %rbp movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 56(%rsp), %rax movq 64(%rsp), %r12 subq $0x418, %rsp @@ -34613,7 +34666,9 @@ _AES_GCM_decrypt_update_avx512: pushq %r15 pushq %rbx movq %rdx, %r10 + movl %esi, %esi movq %rcx, %r11 + movl %r8d, %r8d movq 48(%rsp), %rax movq 56(%rsp), %r12 subq $0x410, %rsp @@ -36268,6 +36323,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_polyval_avx512) .p2align 4 _AES_GCMSIV_polyval_avx512: #endif /* __APPLE__ */ + movl %ecx, %ecx vmovdqu (%rsi), %xmm1 vmovdqu (%rdi), %xmm0 vpshufb L_aes_gcm_siv_bswap_mask_avx512(%rip), %xmm0, %xmm0 @@ -37016,6 +37072,7 @@ WC_ASM_ATT_HIDDEN(_AES_GCMSIV_ctr_avx512) _AES_GCMSIV_ctr_avx512: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vbroadcasti32x4 (%r9), %zmm7 vbroadcasti32x4 256+L_aes_gcmsiv_ctr_inc_avx512(%rip), %zmm8 vbroadcasti32x4 64+L_aes_gcmsiv_ctr_inc_avx512(%rip), %zmm9 diff --git a/wolfcrypt/src/aes_gcm_asm.asm b/wolfcrypt/src/aes_gcm_asm.asm index 1b4b66c7c11..35a6e6dffe2 100644 --- a/wolfcrypt/src/aes_gcm_asm.asm +++ b/wolfcrypt/src/aes_gcm_asm.asm @@ -3822,7 +3822,7 @@ AES_GCM_init_aesni PROC push r13 push r14 mov rdi, rcx - mov rsi, rdx + mov esi, edx mov r10, r8 mov r11d, r9d mov rax, QWORD PTR [rsp+80] @@ -4182,6 +4182,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_aad_update_aesni PROC + mov edx, edx mov rax, rcx sub rsp, 40 movdqu OWORD PTR [rsp+8], xmm6 @@ -4267,6 +4268,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_encrypt_block_aesni PROC + mov edx, edx mov r10, r8 mov r11, r9 mov rax, QWORD PTR [rsp+40] @@ -5645,6 +5647,7 @@ AES_GCM_encrypt_final_aesni PROC push r12 push r14 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+64] @@ -6558,6 +6561,7 @@ AES_GCM_decrypt_final_aesni PROC push rbp push r15 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+80] @@ -6710,6 +6714,7 @@ _TEXT SEGMENT READONLY PARA ENDIF AES_GCMSIV_polyval_aesni PROC push r12 + mov r9d, r9d sub rsp, 144 movdqu OWORD PTR [rsp], xmm6 movdqu OWORD PTR [rsp+16], xmm7 @@ -10519,7 +10524,7 @@ AES_GCM_init_avx1 PROC push r12 push r13 mov rdi, rcx - mov rsi, rdx + mov esi, edx mov r10, r8 mov r11d, r9d mov rax, QWORD PTR [rsp+72] @@ -10844,6 +10849,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_aad_update_avx1 PROC + mov edx, edx mov rax, rcx sub rsp, 40 vmovdqu OWORD PTR [rsp+8], xmm6 @@ -10918,6 +10924,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_encrypt_block_avx1 PROC + mov edx, edx mov r10, r8 mov r11, r9 mov rax, QWORD PTR [rsp+40] @@ -12089,6 +12096,7 @@ AES_GCM_encrypt_final_avx1 PROC push r12 push r14 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+64] @@ -12834,6 +12842,7 @@ AES_GCM_decrypt_final_avx1 PROC push rbp push r15 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+80] @@ -12975,6 +12984,7 @@ _TEXT SEGMENT READONLY PARA ENDIF AES_GCMSIV_polyval_avx1 PROC push r12 + mov r9d, r9d sub rsp, 112 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -16564,7 +16574,7 @@ AES_GCM_init_avx2 PROC push rsi push r12 mov rdi, rcx - mov rsi, rdx + mov esi, edx mov r10, r8 mov r11d, r9d mov rax, QWORD PTR [rsp+72] @@ -16843,6 +16853,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_aad_update_avx2 PROC + mov edx, edx mov rax, rcx sub rsp, 24 vmovdqu OWORD PTR [rsp+8], xmm6 @@ -16903,6 +16914,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_encrypt_block_avx2 PROC + mov edx, edx mov r10, r8 mov r11, r9 mov rax, QWORD PTR [rsp+40] @@ -17905,6 +17917,8 @@ ENDIF AES_GCM_encrypt_final_avx2 PROC push r12 push r13 + mov r8d, r8d + mov r9d, r9d mov eax, DWORD PTR [rsp+56] mov r10, QWORD PTR [rsp+64] mov r11, QWORD PTR [rsp+72] @@ -18521,6 +18535,8 @@ AES_GCM_decrypt_final_avx2 PROC push r12 push r13 push r14 + mov r8d, r8d + mov r9d, r9d mov eax, DWORD PTR [rsp+64] mov r10, QWORD PTR [rsp+72] mov r11, QWORD PTR [rsp+80] @@ -23105,7 +23121,7 @@ AES_GCM_init_vaes PROC push r12 push r13 mov rdi, rcx - mov rsi, rdx + mov esi, edx mov r10, r8 mov r11d, r9d mov rax, QWORD PTR [rsp+72] @@ -23982,6 +23998,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_encrypt_block_vaes PROC + mov edx, edx mov r10, r8 mov r11, r9 mov rax, QWORD PTR [rsp+40] @@ -25083,6 +25100,7 @@ AES_GCM_encrypt_final_vaes PROC push r12 push r14 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+64] @@ -26109,6 +26127,7 @@ AES_GCM_decrypt_final_vaes PROC push rbp push r15 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+80] @@ -26236,6 +26255,7 @@ _TEXT SEGMENT READONLY PARA ENDIF AES_GCMSIV_polyval_vaes PROC push r12 + mov r9d, r9d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -32825,7 +32845,7 @@ AES_GCM_init_avx512 PROC push r12 push r13 mov rdi, rcx - mov rsi, rdx + mov esi, edx mov r10, r8 mov r11d, r9d mov rax, QWORD PTR [rsp+72] @@ -33601,6 +33621,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_GCM_encrypt_block_avx512 PROC + mov edx, edx mov r10, r8 mov r11, r9 mov rax, QWORD PTR [rsp+40] @@ -35500,6 +35521,7 @@ AES_GCM_encrypt_final_avx512 PROC push r12 push r14 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+64] @@ -37170,6 +37192,7 @@ AES_GCM_decrypt_final_avx512 PROC push rbp push r15 mov rax, rcx + mov r8d, r8d mov r10d, r9d mov r9, rdx mov r11d, DWORD PTR [rsp+80] @@ -37298,6 +37321,7 @@ _TEXT SEGMENT READONLY PARA ENDIF AES_GCMSIV_polyval_avx512 PROC push r12 + mov r9d, r9d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 diff --git a/wolfcrypt/src/aes_x86_64_asm.S b/wolfcrypt/src/aes_x86_64_asm.S index 68fe0cfbf92..fe88cbb8401 100644 --- a/wolfcrypt/src/aes_x86_64_asm.S +++ b/wolfcrypt/src/aes_x86_64_asm.S @@ -529,6 +529,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_encrypt_AESNI) .p2align 4 _AES_ECB_encrypt_AESNI: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x40, %edx movl %edx, %r9d @@ -699,6 +700,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_decrypt_AESNI) .p2align 4 _AES_ECB_decrypt_AESNI: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x40, %edx movl %edx, %r9d @@ -869,6 +871,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_encrypt_AESNI) .p2align 4 _AES_CBC_encrypt_AESNI: #endif /* __APPLE__ */ + movl %r9d, %r9d movdqu (%rdx), %xmm0 xorl %eax, %eax cmpl %ecx, %eax @@ -940,6 +943,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_decrypt_AESNI) _AES_CBC_decrypt_AESNI: #endif /* __APPLE__ */ pushq %r12 + movl %r9d, %r9d movdqu (%rdx), %xmm4 xorl %eax, %eax cmpl $0x40, %ecx @@ -1149,6 +1153,7 @@ WC_ASM_ATT_HIDDEN(_AES_CTR_encrypt_AESNI) _AES_CTR_encrypt_AESNI: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d movdqu L_aes_ctr_aesni_bswap(%rip), %xmm8 movdqu L_aes_ctr_aesni_one(%rip), %xmm9 pxor %xmm10, %xmm10 @@ -1373,6 +1378,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_encrypt_avx1) .p2align 4 _AES_ECB_encrypt_avx1: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x40, %edx movl %edx, %r9d @@ -1543,6 +1549,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_decrypt_avx1) .p2align 4 _AES_ECB_decrypt_avx1: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x40, %edx movl %edx, %r9d @@ -1713,6 +1720,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_encrypt_avx1) .p2align 4 _AES_CBC_encrypt_avx1: #endif /* __APPLE__ */ + movl %r9d, %r9d vmovdqu (%rdx), %xmm0 xorl %eax, %eax cmpl %ecx, %eax @@ -1784,6 +1792,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_decrypt_avx1) _AES_CBC_decrypt_avx1: #endif /* __APPLE__ */ pushq %r12 + movl %r9d, %r9d vmovdqu (%rdx), %xmm4 xorl %eax, %eax cmpl $0x40, %ecx @@ -1990,6 +1999,7 @@ WC_ASM_ATT_HIDDEN(_AES_CTR_encrypt_avx1) _AES_CTR_encrypt_avx1: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vmovdqu L_aes_ctr_avx1_bswap(%rip), %xmm8 vmovdqu L_aes_ctr_avx1_one(%rip), %xmm9 vpxor %xmm10, %xmm10, %xmm10 @@ -2200,6 +2210,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_encrypt_vaes) .p2align 4 _AES_ECB_encrypt_vaes: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x80, %edx movl %edx, %r9d @@ -2487,6 +2498,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_decrypt_vaes) .p2align 4 _AES_ECB_decrypt_vaes: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $0x80, %edx movl %edx, %r9d @@ -2774,6 +2786,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_encrypt_vaes) .p2align 4 _AES_CBC_encrypt_vaes: #endif /* __APPLE__ */ + movl %r9d, %r9d vmovdqu (%rdx), %xmm0 xorl %eax, %eax cmpl %ecx, %eax @@ -2845,6 +2858,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_decrypt_vaes) _AES_CBC_decrypt_vaes: #endif /* __APPLE__ */ pushq %r12 + movl %r9d, %r9d vmovdqu (%rdx), %xmm8 xorl %eax, %eax cmpl $0x80, %ecx @@ -3196,6 +3210,7 @@ WC_ASM_ATT_HIDDEN(_AES_CTR_encrypt_vaes) _AES_CTR_encrypt_vaes: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vbroadcasti128 L_aes_ctr_bswap_vaes(%rip), %ymm8 vbroadcasti128 (%r9), %ymm7 vpshufb %ymm8, %ymm7, %ymm7 @@ -3629,6 +3644,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_encrypt_avx512) .p2align 4 _AES_ECB_encrypt_avx512: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $32, %edx jb L_AES_ECB_encrypt_avx512_done_32 @@ -3937,6 +3953,7 @@ WC_ASM_ATT_HIDDEN(_AES_ECB_decrypt_avx512) .p2align 4 _AES_ECB_decrypt_avx512: #endif /* __APPLE__ */ + movl %r8d, %r8d xorl %eax, %eax cmpl $32, %edx jb L_AES_ECB_decrypt_avx512_done_32 @@ -4245,6 +4262,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_encrypt_avx512) .p2align 4 _AES_CBC_encrypt_avx512: #endif /* __APPLE__ */ + movl %r9d, %r9d vmovdqu (%rdx), %xmm0 xorl %eax, %eax cmpl %ecx, %eax @@ -4315,6 +4333,7 @@ WC_ASM_ATT_HIDDEN(_AES_CBC_decrypt_avx512) _AES_CBC_decrypt_avx512: #endif /* __APPLE__ */ pushq %r12 + movl %r9d, %r9d vmovdqu (%rdx), %xmm8 xorl %eax, %eax cmpl $32, %ecx @@ -4693,6 +4712,7 @@ WC_ASM_ATT_HIDDEN(_AES_CTR_encrypt_avx512) _AES_CTR_encrypt_avx512: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vbroadcasti32x4 L_aes_ctr_bswap_avx512(%rip), %zmm8 vbroadcasti32x4 (%r9), %zmm7 vpshufb %zmm8, %zmm7, %zmm7 diff --git a/wolfcrypt/src/aes_xts_asm.S b/wolfcrypt/src/aes_xts_asm.S index a7740ad4375..45ea6ed344b 100644 --- a/wolfcrypt/src/aes_xts_asm.S +++ b/wolfcrypt/src/aes_xts_asm.S @@ -64,6 +64,7 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_init_aesni) .p2align 4 _AES_XTS_init_aesni: #endif /* __APPLE__ */ + movl %edx, %edx movdqu (%rdi), %xmm0 # aes_enc_block pxor (%rsi), %xmm0 @@ -133,7 +134,7 @@ _AES_XTS_encrypt_aesni: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -456,8 +457,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_encrypt_update_aesni) _AES_XTS_encrypt_update_aesni: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp movdqu L_aes_xts_gc_xts(%rip), %xmm12 movdqu (%r8), %xmm0 @@ -744,7 +746,7 @@ _AES_XTS_decrypt_aesni: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $24, %rsp @@ -1121,8 +1123,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_decrypt_update_aesni) _AES_XTS_decrypt_update_aesni: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $16, %rsp movdqu L_aes_xts_gc_xts(%rip), %xmm12 movdqu (%r8), %xmm0 @@ -1462,6 +1465,7 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_init_avx1) .p2align 4 _AES_XTS_init_avx1: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu (%rdi), %xmm0 # aes_enc_block vpxor (%rsi), %xmm0, %xmm0 @@ -1531,7 +1535,7 @@ _AES_XTS_encrypt_avx1: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -1845,8 +1849,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_encrypt_update_avx1) _AES_XTS_encrypt_update_avx1: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp vmovdqu L_avx1_aes_xts_gc_xts(%rip), %xmm12 vmovdqu (%r8), %xmm0 @@ -2124,7 +2129,7 @@ _AES_XTS_decrypt_avx1: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $24, %rsp @@ -2490,8 +2495,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_decrypt_update_avx1) _AES_XTS_decrypt_update_avx1: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $16, %rsp vmovdqu L_avx1_aes_xts_gc_xts(%rip), %xmm12 vmovdqu (%r8), %xmm0 @@ -2821,6 +2827,7 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_init_vaes) .p2align 4 _AES_XTS_init_vaes: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu (%rdi), %xmm0 # aes_enc_block vpxor (%rsi), %xmm0, %xmm0 @@ -2928,7 +2935,7 @@ _AES_XTS_encrypt_vaes: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -3429,8 +3436,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_encrypt_update_vaes) _AES_XTS_encrypt_update_vaes: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp vmovdqu L_vaes_aes_xts_gc_xts(%rip), %xmm12 vbroadcasti128 L_vaes_aes_xts_poly(%rip), %ymm13 @@ -3895,7 +3903,7 @@ _AES_XTS_decrypt_vaes: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -4470,8 +4478,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_decrypt_update_vaes) _AES_XTS_decrypt_update_vaes: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp vmovdqu L_vaes_aes_xts_gc_xts(%rip), %xmm12 vbroadcasti128 L_vaes_aes_xts_poly(%rip), %ymm13 @@ -5010,6 +5019,7 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_init_avx512) .p2align 4 _AES_XTS_init_avx512: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu (%rdi), %xmm0 # aes_enc_block vpxor (%rsi), %xmm0, %xmm0 @@ -5121,7 +5131,7 @@ _AES_XTS_encrypt_avx512: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -5629,8 +5639,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_encrypt_update_avx512) _AES_XTS_encrypt_update_avx512: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp vmovdqu L_avx512_aes_xts_gc_xts(%rip), %xmm12 vbroadcasti32x4 L_avx512_aes_xts_poly(%rip), %zmm13 @@ -6102,7 +6113,7 @@ _AES_XTS_decrypt_avx512: #endif /* __APPLE__ */ pushq %r12 pushq %r13 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r12 movl 24(%rsp), %r10d subq $0x48, %rsp @@ -6694,8 +6705,9 @@ WC_ASM_ATT_HIDDEN(_AES_XTS_decrypt_update_avx512) _AES_XTS_decrypt_update_avx512: #endif /* __APPLE__ */ pushq %r12 - movq %rdx, %rax + movl %edx, %eax movq %rcx, %r10 + movl %r9d, %r9d subq $0x40, %rsp vmovdqu L_avx512_aes_xts_gc_xts(%rip), %xmm12 vbroadcasti32x4 L_avx512_aes_xts_poly(%rip), %zmm13 diff --git a/wolfcrypt/src/aes_xts_asm.asm b/wolfcrypt/src/aes_xts_asm.asm index f561612f9a9..25cc69721d1 100644 --- a/wolfcrypt/src/aes_xts_asm.asm +++ b/wolfcrypt/src/aes_xts_asm.asm @@ -60,6 +60,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_XTS_init_aesni PROC + mov r8d, r8d movdqu xmm0, OWORD PTR [rcx] ; aes_enc_block pxor xmm0, [rdx] @@ -131,7 +132,7 @@ AES_XTS_encrypt_aesni PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -471,7 +472,7 @@ AES_XTS_encrypt_update_aesni PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -776,7 +777,7 @@ AES_XTS_decrypt_aesni PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -1170,7 +1171,7 @@ AES_XTS_decrypt_update_aesni PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -1524,6 +1525,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_XTS_init_avx1 PROC + mov r8d, r8d vmovdqu xmm0, OWORD PTR [rcx] ; aes_enc_block vpxor xmm0, xmm0, [rdx] @@ -1595,7 +1597,7 @@ AES_XTS_encrypt_avx1 PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -1926,7 +1928,7 @@ AES_XTS_encrypt_update_avx1 PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -2222,7 +2224,7 @@ AES_XTS_decrypt_avx1 PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -2605,7 +2607,7 @@ AES_XTS_decrypt_update_avx1 PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -2949,6 +2951,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_XTS_init_vaes PROC + mov r8d, r8d vmovdqu xmm0, OWORD PTR [rcx] ; aes_enc_block vpxor xmm0, xmm0, [rdx] @@ -3067,7 +3070,7 @@ AES_XTS_encrypt_vaes PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -3591,7 +3594,7 @@ AES_XTS_encrypt_update_vaes PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -4080,7 +4083,7 @@ AES_XTS_decrypt_vaes PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -4678,7 +4681,7 @@ AES_XTS_decrypt_update_vaes PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -5237,6 +5240,7 @@ ELSE _TEXT SEGMENT READONLY PARA ENDIF AES_XTS_init_avx512 PROC + mov r8d, r8d vmovdqu xmm0, OWORD PTR [rcx] ; aes_enc_block vpxor xmm0, xmm0, [rdx] @@ -5359,7 +5363,7 @@ AES_XTS_encrypt_avx512 PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -5890,7 +5894,7 @@ AES_XTS_encrypt_update_avx512 PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] @@ -6386,7 +6390,7 @@ AES_XTS_decrypt_avx512 PROC push r13 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r12, r9 mov r8, QWORD PTR [rsp+72] mov r9, QWORD PTR [rsp+80] @@ -7001,7 +7005,7 @@ AES_XTS_decrypt_update_avx512 PROC push r12 mov rdi, rcx mov rsi, rdx - mov rax, r8 + mov eax, r8d mov r10, r9 mov r8, QWORD PTR [rsp+64] mov r9d, DWORD PTR [rsp+72] diff --git a/wolfcrypt/src/chacha.c b/wolfcrypt/src/chacha.c index fa3a04b0a86..bf9eefb8bdb 100644 --- a/wolfcrypt/src/chacha.c +++ b/wolfcrypt/src/chacha.c @@ -389,6 +389,18 @@ static WC_INLINE int chacha_avx512_beneficial(cpuid_flags_t flags) } #endif /* USE_INTEL_CHACHA_SPEEDUP && HAVE_INTEL_AVX512 */ +#ifdef USE_INTEL_CHACHA_SPEEDUP +/* Whether the AVX-512VL kernel handles short input on this CPU. It is only + * built with HAVE_INTEL_AVX512: without it (e.g. NO_AVX512_SUPPORT on MSVC) the + * CPUID bit must not steer short input away from the AVX1/SSSE3 paths, as + * nothing would pick it up but the scalar code. */ +#ifdef HAVE_INTEL_AVX512 + #define CHACHA_USE_AVX512_VL(f) (IS_INTEL_AVX512_VL(f) != 0) +#else + #define CHACHA_USE_AVX512_VL(f) 0 +#endif +#endif /* USE_INTEL_CHACHA_SPEEDUP */ + #if (!defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP) && \ !defined(USE_RISCV_CHACHA_SPEEDUP)) || defined(WOLFSSL_ARM_CHACHA_NEED_C) @@ -497,7 +509,7 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input, * cyc) beats the scalar block (~435) - e.g. the per-record Poly1305 key * derivation (a 32-byte ChaCha) in the ChaCha20-Poly1305 two-pass path. * The AVX-512VL path already uses SIMD for one block; match that here. */ - if (msglen <= CHACHA_CHUNK_BYTES && IS_INTEL_AVX512_VL(cpuidFlags) == 0 && + if (msglen <= CHACHA_CHUNK_BYTES && !CHACHA_USE_AVX512_VL(cpuidFlags) && IS_INTEL_AVX1(cpuidFlags)) { SAVE_VECTOR_REGISTERS(return _svr_ret;); chacha_encrypt_avx1(ctx, input, output, msglen); @@ -521,7 +533,7 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input, * key + <=2 data blocks). At >=256 bytes the four-block AVX2/AVX1 kernels * take over below. */ #ifdef HAVE_INTEL_SSSE3 - if (IS_INTEL_AVX512_VL(cpuidFlags) == 0 && + if (!CHACHA_USE_AVX512_VL(cpuidFlags) && msglen < 4 * CHACHA_CHUNK_BYTES && IS_INTEL_SSSE3(cpuidFlags)) { SAVE_VECTOR_REGISTERS(return _svr_ret;); @@ -530,7 +542,7 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input, return 0; } #endif - if (IS_INTEL_AVX512_VL(cpuidFlags) == 0 && + if (!CHACHA_USE_AVX512_VL(cpuidFlags) && msglen < 4 * CHACHA_CHUNK_BYTES) { chacha_encrypt_x64(ctx, input, output, msglen); return 0; @@ -551,7 +563,7 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input, * partial four-block tail - using single-instruction vprold rotations on * 128-bit registers (no AVX-512 frequency penalty). It does not fall back * to any other implementation. */ - if (IS_INTEL_AVX512_VL(cpuidFlags) && msglen < 8 * CHACHA_CHUNK_BYTES) { + if (CHACHA_USE_AVX512_VL(cpuidFlags) && msglen < 8 * CHACHA_CHUNK_BYTES) { SAVE_VECTOR_REGISTERS(return _svr_ret;); chacha_encrypt_avx512vl(ctx, input, output, msglen); RESTORE_VECTOR_REGISTERS(); diff --git a/wolfcrypt/src/chacha_asm.S b/wolfcrypt/src/chacha_asm.S index f4ef1cfc5f0..f2af55ca1e2 100644 --- a/wolfcrypt/src/chacha_asm.S +++ b/wolfcrypt/src/chacha_asm.S @@ -68,6 +68,7 @@ _chacha_encrypt_x64: pushq %r13 pushq %r14 pushq %r15 + movl %ecx, %ecx subq $0x48, %rsp cmpl $0x40, %ecx jb L_chacha_x64_small @@ -511,6 +512,7 @@ _chacha_encrypt_sse3: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %ecx, %ecx leaq L_chacha20_sse3_rotl8(%rip), %r11 leaq L_chacha20_sse3_rotl16(%rip), %r12 leaq L_chacha20_sse3_one(%rip), %r13 @@ -897,6 +899,7 @@ _chacha_encrypt_avx1: pushq %r13 pushq %r14 pushq %r15 + movl %ecx, %ecx subq $0x198, %rsp movq %rsp, %r9 leaq 256(%rsp), %r10 @@ -1470,6 +1473,7 @@ _chacha_encrypt_avx2: pushq %r12 pushq %r13 pushq %r14 + movl %ecx, %ecx subq $0x320, %rsp movq %rsp, %r9 leaq L_chacha20_avx2_rotl8(%rip), %r11 @@ -1873,6 +1877,7 @@ _chacha_encrypt_avx512vl: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %ecx, %ecx subq $0x198, %rsp movq %rsp, %r9 leaq 256(%rsp), %r10 @@ -2472,6 +2477,7 @@ WC_ASM_ATT_HIDDEN(_chacha_encrypt_avx512) .p2align 4 _chacha_encrypt_avx512: #endif /* __APPLE__ */ + movl %ecx, %ecx subq $8, %rsp leaq L_chacha20_avx512_add(%rip), %r8 cmpl $0x400, %ecx @@ -2798,6 +2804,7 @@ _chacha20_poly1305_avx512: pushq %r15 pushq %rbx pushq %rbp + movl %r8d, %r8d subq $0x2d8, %rsp leaq L_chacha20_poly1305_add(%rip), %r10 leaq L_chacha20_poly1305_four(%rip), %r11 @@ -5014,6 +5021,7 @@ _chacha20_poly1305_ifma: pushq %r12 pushq %r13 pushq %r14 + movl %r8d, %r8d subq $0x4b0, %rsp leaq L_cp512_pc(%rip), %rax leaq L_cp512_rx(%rip), %r10 @@ -8315,6 +8323,7 @@ _chacha20_poly1305_ifma_decrypt: pushq %r12 pushq %r13 pushq %r14 + movl %r8d, %r8d subq $0x4b0, %rsp leaq L_cp512_pc(%rip), %rax leaq L_cp512_rx(%rip), %r10 @@ -10106,6 +10115,7 @@ _chacha20_poly1305_small_enc: pushq %r15 pushq %rbx pushq %rbp + movl %r8d, %r8d subq $0xc8, %rsp movl 256(%rsp), %eax movq %rax, 128(%rsp) @@ -10586,6 +10596,7 @@ _chacha20_poly1305_small_dec: pushq %r15 pushq %rbx pushq %rbp + movl %r8d, %r8d subq $0xa8, %rsp movl 224(%rsp), %eax movq %rax, 128(%rsp) diff --git a/wolfcrypt/src/chacha_asm.asm b/wolfcrypt/src/chacha_asm.asm index 05a526504aa..acbdcbc652a 100644 --- a/wolfcrypt/src/chacha_asm.asm +++ b/wolfcrypt/src/chacha_asm.asm @@ -54,6 +54,9 @@ chacha_encrypt_x64 PROC push r13 push r14 push r15 + push rdi + push rsi + mov r9d, r9d sub rsp, 72 cmp r9d, 64 jb L_chacha_x64_small @@ -70,8 +73,8 @@ L_chacha_x64_start: mov ebx, DWORD PTR [rcx+4] mov r9d, DWORD PTR [rcx+8] mov r8d, DWORD PTR [rcx+12] - mov r8d, DWORD PTR [rcx+16] - mov r9d, DWORD PTR [rcx+20] + mov edi, DWORD PTR [rcx+16] + mov esi, DWORD PTR [rcx+20] mov r10d, DWORD PTR [rcx+24] mov r11d, DWORD PTR [rcx+28] mov r12d, DWORD PTR [rcx+48] @@ -80,8 +83,8 @@ L_chacha_x64_start: mov r15d, DWORD PTR [rcx+60] mov BYTE PTR [rsp], 10 L_chacha_x64_block_crypt_start: - add eax, r8d - add ebx, r9d + add eax, edi + add ebx, esi add r9d, r10d add r8d, r11d xor r12d, eax @@ -96,16 +99,16 @@ L_chacha_x64_block_crypt_start: add DWORD PTR [rsp+12], r13d add DWORD PTR [rsp+16], r14d add DWORD PTR [rsp+20], r15d - xor r8d, DWORD PTR [rsp+8] - xor r9d, DWORD PTR [rsp+12] + xor edi, DWORD PTR [rsp+8] + xor esi, DWORD PTR [rsp+12] xor r10d, DWORD PTR [rsp+16] xor r11d, DWORD PTR [rsp+20] - rol r8d, 12 - rol r9d, 12 + rol edi, 12 + rol esi, 12 rol r10d, 12 rol r11d, 12 - add eax, r8d - add ebx, r9d + add eax, edi + add ebx, esi add r9d, r10d add r8d, r11d xor r12d, eax @@ -120,18 +123,18 @@ L_chacha_x64_block_crypt_start: add DWORD PTR [rsp+12], r13d add DWORD PTR [rsp+16], r14d add DWORD PTR [rsp+20], r15d - xor r8d, DWORD PTR [rsp+8] - xor r9d, DWORD PTR [rsp+12] + xor edi, DWORD PTR [rsp+8] + xor esi, DWORD PTR [rsp+12] xor r10d, DWORD PTR [rsp+16] xor r11d, DWORD PTR [rsp+20] - rol r8d, 7 - rol r9d, 7 + rol edi, 7 + rol esi, 7 rol r10d, 7 rol r11d, 7 - add eax, r9d + add eax, esi add ebx, r10d add r9d, r11d - add r8d, r8d + add r8d, edi xor r15d, eax xor r12d, ebx xor r13d, r9d @@ -144,18 +147,18 @@ L_chacha_x64_block_crypt_start: add DWORD PTR [rsp+20], r12d add DWORD PTR [rsp+8], r13d add DWORD PTR [rsp+12], r14d - xor r9d, DWORD PTR [rsp+16] + xor esi, DWORD PTR [rsp+16] xor r10d, DWORD PTR [rsp+20] xor r11d, DWORD PTR [rsp+8] - xor r8d, DWORD PTR [rsp+12] - rol r9d, 12 + xor edi, DWORD PTR [rsp+12] + rol esi, 12 rol r10d, 12 rol r11d, 12 - rol r8d, 12 - add eax, r9d + rol edi, 12 + add eax, esi add ebx, r10d add r9d, r11d - add r8d, r8d + add r8d, edi xor r15d, eax xor r12d, ebx xor r13d, r9d @@ -168,14 +171,14 @@ L_chacha_x64_block_crypt_start: add DWORD PTR [rsp+20], r12d add DWORD PTR [rsp+8], r13d add DWORD PTR [rsp+12], r14d - xor r9d, DWORD PTR [rsp+16] + xor esi, DWORD PTR [rsp+16] xor r10d, DWORD PTR [rsp+20] xor r11d, DWORD PTR [rsp+8] - xor r8d, DWORD PTR [rsp+12] - rol r9d, 7 + xor edi, DWORD PTR [rsp+12] + rol esi, 7 rol r10d, 7 rol r11d, 7 - rol r8d, 7 + rol edi, 7 dec BYTE PTR [rsp] jnz L_chacha_x64_block_crypt_start mov rdx, QWORD PTR [rsp+32] @@ -184,8 +187,8 @@ L_chacha_x64_block_crypt_start: add ebx, DWORD PTR [rcx+4] add r9d, DWORD PTR [rcx+8] add r8d, DWORD PTR [rcx+12] - add r8d, DWORD PTR [rcx+16] - add r9d, DWORD PTR [rcx+20] + add edi, DWORD PTR [rcx+16] + add esi, DWORD PTR [rcx+20] add r10d, DWORD PTR [rcx+24] add r11d, DWORD PTR [rcx+28] add r12d, DWORD PTR [rcx+48] @@ -196,8 +199,8 @@ L_chacha_x64_block_crypt_start: xor ebx, DWORD PTR [rdx+4] xor r9d, DWORD PTR [rdx+8] xor r8d, DWORD PTR [rdx+12] - xor r8d, DWORD PTR [rdx+16] - xor r9d, DWORD PTR [rdx+20] + xor edi, DWORD PTR [rdx+16] + xor esi, DWORD PTR [rdx+20] xor r10d, DWORD PTR [rdx+24] xor r11d, DWORD PTR [rdx+28] xor r12d, DWORD PTR [rdx+48] @@ -208,8 +211,8 @@ L_chacha_x64_block_crypt_start: mov DWORD PTR [rbp+4], ebx mov DWORD PTR [rbp+8], r9d mov DWORD PTR [rbp+12], r8d - mov DWORD PTR [rbp+16], r8d - mov DWORD PTR [rbp+20], r9d + mov DWORD PTR [rbp+16], edi + mov DWORD PTR [rbp+20], esi mov DWORD PTR [rbp+24], r10d mov DWORD PTR [rbp+28], r11d mov DWORD PTR [rbp+48], r12d @@ -256,8 +259,8 @@ L_chacha_x64_small: mov ebx, DWORD PTR [rcx+4] mov r9d, DWORD PTR [rcx+8] mov r8d, DWORD PTR [rcx+12] - mov r8d, DWORD PTR [rcx+16] - mov r9d, DWORD PTR [rcx+20] + mov edi, DWORD PTR [rcx+16] + mov esi, DWORD PTR [rcx+20] mov r10d, DWORD PTR [rcx+24] mov r11d, DWORD PTR [rcx+28] mov r12d, DWORD PTR [rcx+48] @@ -266,8 +269,8 @@ L_chacha_x64_small: mov r15d, DWORD PTR [rcx+60] mov BYTE PTR [rsp], 10 L_chacha_x64_partial_crypt_start: - add eax, r8d - add ebx, r9d + add eax, edi + add ebx, esi add r9d, r10d add r8d, r11d xor r12d, eax @@ -282,16 +285,16 @@ L_chacha_x64_partial_crypt_start: add DWORD PTR [rsp+12], r13d add DWORD PTR [rsp+16], r14d add DWORD PTR [rsp+20], r15d - xor r8d, DWORD PTR [rsp+8] - xor r9d, DWORD PTR [rsp+12] + xor edi, DWORD PTR [rsp+8] + xor esi, DWORD PTR [rsp+12] xor r10d, DWORD PTR [rsp+16] xor r11d, DWORD PTR [rsp+20] - rol r8d, 12 - rol r9d, 12 + rol edi, 12 + rol esi, 12 rol r10d, 12 rol r11d, 12 - add eax, r8d - add ebx, r9d + add eax, edi + add ebx, esi add r9d, r10d add r8d, r11d xor r12d, eax @@ -306,18 +309,18 @@ L_chacha_x64_partial_crypt_start: add DWORD PTR [rsp+12], r13d add DWORD PTR [rsp+16], r14d add DWORD PTR [rsp+20], r15d - xor r8d, DWORD PTR [rsp+8] - xor r9d, DWORD PTR [rsp+12] + xor edi, DWORD PTR [rsp+8] + xor esi, DWORD PTR [rsp+12] xor r10d, DWORD PTR [rsp+16] xor r11d, DWORD PTR [rsp+20] - rol r8d, 7 - rol r9d, 7 + rol edi, 7 + rol esi, 7 rol r10d, 7 rol r11d, 7 - add eax, r9d + add eax, esi add ebx, r10d add r9d, r11d - add r8d, r8d + add r8d, edi xor r15d, eax xor r12d, ebx xor r13d, r9d @@ -330,18 +333,18 @@ L_chacha_x64_partial_crypt_start: add DWORD PTR [rsp+20], r12d add DWORD PTR [rsp+8], r13d add DWORD PTR [rsp+12], r14d - xor r9d, DWORD PTR [rsp+16] + xor esi, DWORD PTR [rsp+16] xor r10d, DWORD PTR [rsp+20] xor r11d, DWORD PTR [rsp+8] - xor r8d, DWORD PTR [rsp+12] - rol r9d, 12 + xor edi, DWORD PTR [rsp+12] + rol esi, 12 rol r10d, 12 rol r11d, 12 - rol r8d, 12 - add eax, r9d + rol edi, 12 + add eax, esi add ebx, r10d add r9d, r11d - add r8d, r8d + add r8d, edi xor r15d, eax xor r12d, ebx xor r13d, r9d @@ -354,14 +357,14 @@ L_chacha_x64_partial_crypt_start: add DWORD PTR [rsp+20], r12d add DWORD PTR [rsp+8], r13d add DWORD PTR [rsp+12], r14d - xor r9d, DWORD PTR [rsp+16] + xor esi, DWORD PTR [rsp+16] xor r10d, DWORD PTR [rsp+20] xor r11d, DWORD PTR [rsp+8] - xor r8d, DWORD PTR [rsp+12] - rol r9d, 7 + xor edi, DWORD PTR [rsp+12] + rol esi, 7 rol r10d, 7 rol r11d, 7 - rol r8d, 7 + rol edi, 7 dec BYTE PTR [rsp] jnz L_chacha_x64_partial_crypt_start mov rdx, QWORD PTR [rsp+32] @@ -369,8 +372,8 @@ L_chacha_x64_partial_crypt_start: add ebx, DWORD PTR [rcx+4] add r9d, DWORD PTR [rcx+8] add r8d, DWORD PTR [rcx+12] - add r8d, DWORD PTR [rcx+16] - add r9d, DWORD PTR [rcx+20] + add edi, DWORD PTR [rcx+16] + add esi, DWORD PTR [rcx+20] add r10d, DWORD PTR [rcx+24] add r11d, DWORD PTR [rcx+28] add r12d, DWORD PTR [rcx+48] @@ -382,8 +385,8 @@ L_chacha_x64_partial_crypt_start: mov DWORD PTR [rbp+4], ebx mov DWORD PTR [rbp+8], r9d mov DWORD PTR [rbp+12], r8d - mov DWORD PTR [rbp+16], r8d - mov DWORD PTR [rbp+20], r9d + mov DWORD PTR [rbp+16], edi + mov DWORD PTR [rbp+20], esi mov DWORD PTR [rbp+24], r10d mov DWORD PTR [rbp+28], r11d mov DWORD PTR [rbp+48], r12d @@ -406,16 +409,16 @@ L_chacha_x64_partial_crypt_start: mov r9, QWORD PTR [rsp+40] add DWORD PTR [rcx+48], 1 add rsp, 48 - mov r8d, r9d + mov edi, r9d xor rbx, rbx - and r8d, 7 + and edi, 7 jz L_chacha_x64_partial_start64 L_chacha_x64_partial_start8: movzx eax, BYTE PTR [rbp+rbx] xor al, BYTE PTR [rdx+rbx] mov BYTE PTR [r8+rbx], al inc ebx - cmp ebx, r8d + cmp ebx, edi jne L_chacha_x64_partial_start8 je L_chacha_x64_partial_end64 L_chacha_x64_partial_start64: @@ -431,6 +434,8 @@ L_chacha_x64_partial_end64: mov DWORD PTR [rcx+76], r9d L_chacha_x64_done: add rsp, 72 + pop rsi + pop rdi pop r15 pop r14 pop r13 @@ -465,6 +470,7 @@ chacha_encrypt_sse3 PROC push r13 push r14 push r15 + mov r9d, r9d sub rsp, 168 movdqu OWORD PTR [rsp+8], xmm6 movdqu OWORD PTR [rsp+24], xmm7 @@ -836,6 +842,7 @@ chacha_encrypt_avx1 PROC push r15 push rdi push rsi + mov r9d, r9d sub rsp, 568 vmovdqu OWORD PTR [rsp+408], xmm6 vmovdqu OWORD PTR [rsp+424], xmm7 @@ -1392,6 +1399,7 @@ chacha_encrypt_avx2 PROC push r14 push r15 push rdi + mov r9d, r9d sub rsp, 960 vmovdqu OWORD PTR [rsp+800], xmm6 vmovdqu OWORD PTR [rsp+816], xmm7 @@ -1790,6 +1798,7 @@ chacha_encrypt_avx512vl PROC push r13 push r14 push r15 + mov r9d, r9d sub rsp, 568 vmovdqu OWORD PTR [rsp+408], xmm6 vmovdqu OWORD PTR [rsp+424], xmm7 @@ -2391,6 +2400,7 @@ ptr_L_chacha20_avx512_add QWORD L_chacha20_avx512_add _DATA ENDS _TEXT SEGMENT READONLY PARA chacha_encrypt_avx512 PROC + mov r9d, r9d sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -2700,7 +2710,7 @@ chacha20_poly1305_avx512 PROC mov rsi, rdx mov rdx, r8 mov rcx, r9 - mov rax, QWORD PTR [rsp+104] + mov eax, DWORD PTR [rsp+104] sub rsp, 888 vmovdqu OWORD PTR [rsp+728], xmm6 vmovdqu OWORD PTR [rsp+744], xmm7 @@ -4911,7 +4921,7 @@ chacha20_poly1305_ifma PROC mov rsi, rdx mov rdx, r8 mov rcx, r9 - mov rax, QWORD PTR [rsp+80] + mov eax, DWORD PTR [rsp+80] sub rsp, 1360 vmovdqu OWORD PTR [rsp+1200], xmm6 vmovdqu OWORD PTR [rsp+1216], xmm7 @@ -8228,7 +8238,7 @@ chacha20_poly1305_ifma_decrypt PROC mov rsi, rdx mov rdx, r8 mov rcx, r9 - mov rax, QWORD PTR [rsp+80] + mov eax, DWORD PTR [rsp+80] sub rsp, 1360 vmovdqu OWORD PTR [rsp+1200], xmm6 vmovdqu OWORD PTR [rsp+1216], xmm7 @@ -10010,7 +10020,7 @@ chacha20_poly1305_small_enc PROC push rsi push rbx push rbp - mov r10, QWORD PTR [rsp+104] + mov r10d, DWORD PTR [rsp+104] mov r11, QWORD PTR [rsp+112] sub rsp, 360 vmovdqu OWORD PTR [rsp+200], xmm6 @@ -10503,7 +10513,7 @@ chacha20_poly1305_small_dec PROC push rsi push rbx push rbp - mov r10, QWORD PTR [rsp+104] + mov r10d, DWORD PTR [rsp+104] mov r11, QWORD PTR [rsp+112] sub rsp, 328 vmovdqu OWORD PTR [rsp+168], xmm6 diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c index bd20dea294f..e6d3d048cb4 100644 --- a/wolfcrypt/src/cpuid.c +++ b/wolfcrypt/src/cpuid.c @@ -174,6 +174,45 @@ } +#ifndef WC_CPUID_NO_TUNING + /* Return the tuning bits (CPUID_SLOW_*) for an Intel CPU, chosen from the + * family and display model in CPUID.1:EAX. */ + static cpuid_flags_t cpuid_intel_tuning(void) + { + unsigned int reg[5]; + word32 family; + word32 model; + + XMEMSET(reg, '\0', sizeof(reg)); + cpuid(reg, 1, 0); + + family = (reg[EAX] >> 8) & 0xf; + model = (reg[EAX] >> 4) & 0xf; + if ((family == 0x6) || (family == 0xf)) { + model |= ((reg[EAX] >> 16) & 0xf) << 4; + } + if (family == 0xf) { + family += (reg[EAX] >> 20) & 0xff; + } + + if (family == 0x6) { + switch (model) { + /* Arrow Lake and Lunar Lake (Lion Cove / Skymont): VAES AES + * is no faster than AVX2, and the AVX2 Keccak block is slower + * than BMI2 and C. */ + case 0xB5: /* Arrow Lake U */ + case 0xC5: /* Arrow Lake H */ + case 0xC6: /* Arrow Lake S */ + case 0xBD: /* Lunar Lake M */ + return CPUID_SLOW_VAES | CPUID_SLOW_AVX2_SHA3; + default: + break; + } + } + return 0; + } +#endif + static WC_INLINE void cpuid_set_flags(void) { if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { @@ -222,6 +261,11 @@ if (cpuid_is_intel()) { new_cpuid_flags |= CPUID_INTEL ; } if (cpuid_is_amd()) { new_cpuid_flags |= CPUID_AMD ; } if (cpuid_flag(1, 0, ECX, 9)) { new_cpuid_flags |= CPUID_SSSE3 ; } + #ifndef WC_CPUID_NO_TUNING + if (cpuid_is_intel()) { + new_cpuid_flags |= cpuid_intel_tuning(); + } + #endif (void)wolfSSL_Atomic_Uint_CompareExchange (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); } diff --git a/wolfcrypt/src/fe_x25519_asm.S b/wolfcrypt/src/fe_x25519_asm.S index 18f5e678ae6..fc960dc5a4b 100644 --- a/wolfcrypt/src/fe_x25519_asm.S +++ b/wolfcrypt/src/fe_x25519_asm.S @@ -673,6 +673,7 @@ WC_ASM_ATT_HIDDEN(_fe_cmov) .p2align 4 _fe_cmov: #endif /* __APPLE__ */ + movl %edx, %edx cmpl $0x01, %edx movq (%rdi), %rcx movq 8(%rdi), %r8 diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index aab90b8ddd3..efc2ed5a00b 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -314,6 +314,7 @@ _TEXT ENDS _TEXT SEGMENT READONLY PARA fe_cmov PROC push r12 + mov r8d, r8d cmp r8d, 1 mov r9, QWORD PTR [rcx] mov r10, QWORD PTR [rcx+8] diff --git a/wolfcrypt/src/sha256_asm.S b/wolfcrypt/src/sha256_asm.S index 19609aea285..2dc6a8e3c79 100644 --- a/wolfcrypt/src/sha256_asm.S +++ b/wolfcrypt/src/sha256_asm.S @@ -301,6 +301,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_SSE2_Sha_Len) .p2align 4 _Transform_Sha256_SSE2_Sha_Len: #endif /* __APPLE__ */ + movl %edx, %edx leaq L_sse2_sha256_sha_k(%rip), %rax movdqa L_sse2_sha256_shuf_mask(%rip), %xmm10 movq (%rdi), %xmm1 @@ -2968,7 +2969,7 @@ _Transform_Sha256_AVX1_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rdx, %rbp + movl %edx, %ebp subq $0x48, %rsp movl %ebp, 64(%rsp) leaq L_avx1_sha256_k(%rip), %rbp @@ -7790,7 +7791,7 @@ _Transform_Sha256_AVX1_RORX_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rdx, %rbp + movl %edx, %ebp subq $0x48, %rsp movl %ebp, 64(%rsp) leaq L_avx1_rorx_sha256_k(%rip), %rbp @@ -10361,6 +10362,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_AVX1_Sha_Len) .p2align 4 _Transform_Sha256_AVX1_Sha_Len: #endif /* __APPLE__ */ + movl %edx, %edx leaq L_avx1_sha256_sha_k(%rip), %rax vmovdqa L_avx1_sha256_shuf_mask(%rip), %xmm10 vmovq (%rdi), %xmm1 @@ -13021,7 +13023,7 @@ _Transform_Sha256_AVX2_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rdx, %rbp + movl %edx, %ebp subq $0x208, %rsp testb $0x40, %bpl movl %ebp, 512(%rsp) @@ -19604,7 +19606,7 @@ _Transform_Sha256_AVX2_RORX_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rdx, %rbp + movl %edx, %ebp subq $0x208, %rsp testb $0x40, %bpl movl %ebp, 512(%rsp) @@ -27669,6 +27671,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_x16_LmsInit_AVX512) _Transform_Sha256_x16_LmsInit_AVX512: #endif /* __APPLE__ */ pushq %rbp + movl %edx, %edx leaq L_avx512_sha256_x16_k(%rip), %rbp leaq L_avx512_sha256_x16_iv(%rip), %rcx # W from the template, all lanes alike but for the chain index @@ -29563,6 +29566,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_x16_LmsChain_AVX512) _Transform_Sha256_x16_LmsChain_AVX512: #endif /* __APPLE__ */ pushq %rbp + movl %ecx, %ecx testl %ecx, %ecx jz L_sha256_x16_lms_chain_done leaq L_avx512_sha256_x16_iv(%rip), %r8 @@ -30534,6 +30538,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_x16_Lms192Init_AVX512) _Transform_Sha256_x16_Lms192Init_AVX512: #endif /* __APPLE__ */ pushq %rbp + movl %edx, %edx leaq L_avx512_sha256_x16_k(%rip), %rbp leaq L_avx512_sha256_x16_iv(%rip), %rcx # W from the template, all lanes alike but for the chain index @@ -32410,6 +32415,7 @@ WC_ASM_ATT_HIDDEN(_Transform_Sha256_x16_Lms192Chain_AVX512) _Transform_Sha256_x16_Lms192Chain_AVX512: #endif /* __APPLE__ */ pushq %rbp + movl %ecx, %ecx testl %ecx, %ecx jz L_sha256_x16_lms192_chain_done leaq L_avx512_sha256_x16_iv(%rip), %r8 diff --git a/wolfcrypt/src/sha256_asm.asm b/wolfcrypt/src/sha256_asm.asm index 46043c932fe..23bd0d91ace 100644 --- a/wolfcrypt/src/sha256_asm.asm +++ b/wolfcrypt/src/sha256_asm.asm @@ -272,6 +272,7 @@ Transform_Sha256_SSE2_Sha ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA Transform_Sha256_SSE2_Sha_Len PROC + mov r8d, r8d sub rsp, 88 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -2923,7 +2924,7 @@ Transform_Sha256_AVX1_Len PROC push rbp mov rdi, rcx mov rsi, rdx - mov rbp, r8 + mov ebp, r8d sub rsp, 200 vmovdqu OWORD PTR [rsp+72], xmm6 vmovdqu OWORD PTR [rsp+88], xmm7 @@ -7735,7 +7736,7 @@ Transform_Sha256_AVX1_RORX_Len PROC push rbp mov rdi, rcx mov rsi, rdx - mov rbp, r8 + mov ebp, r8d sub rsp, 200 vmovdqu OWORD PTR [rsp+72], xmm6 vmovdqu OWORD PTR [rsp+88], xmm7 @@ -10296,6 +10297,7 @@ Transform_Sha256_AVX1_Sha ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA Transform_Sha256_AVX1_Sha_Len PROC + mov r8d, r8d sub rsp, 88 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -12940,7 +12942,7 @@ Transform_Sha256_AVX2_Len PROC push rbp mov rdi, rcx mov rsi, rdx - mov rbp, r8 + mov ebp, r8d sub rsp, 648 vmovdqu OWORD PTR [rsp+520], xmm6 vmovdqu OWORD PTR [rsp+536], xmm7 @@ -19511,7 +19513,7 @@ Transform_Sha256_AVX2_RORX_Len PROC push rbp mov rdi, rcx mov rsi, rdx - mov rbp, r8 + mov ebp, r8d sub rsp, 648 vmovdqu OWORD PTR [rsp+520], xmm6 vmovdqu OWORD PTR [rsp+536], xmm7 diff --git a/wolfcrypt/src/sha3_asm.S b/wolfcrypt/src/sha3_asm.S index b706156983c..7ca32e928ea 100644 --- a/wolfcrypt/src/sha3_asm.S +++ b/wolfcrypt/src/sha3_asm.S @@ -4631,6 +4631,7 @@ _sha3_block_n_bmi2: pushq %r15 pushq %rbx pushq %rbp + movl %edx, %edx pushq %rcx movq %rcx, %rbp movq (%rdi), %rcx @@ -9594,6 +9595,7 @@ WC_ASM_ATT_HIDDEN(_sha3_block_n_avx2) .p2align 4 _sha3_block_n_avx2: #endif /* __APPLE__ */ + movl %edx, %edx leaq L_sha3_avx2_r(%rip), %rax leaq L_sha3_block_n_avx2_rotl(%rip), %r8 addq $0x40, %r8 @@ -15366,6 +15368,7 @@ WC_ASM_ATT_HIDDEN(_sha3_xmss_blocksx4_avx2) .p2align 4 _sha3_xmss_blocksx4_avx2: #endif /* __APPLE__ */ + movl %r8d, %r8d leaq L_sha3_x4_avx2_r(%rip), %rax movq %rdi, %r10 movq %rdi, %r11 @@ -20885,6 +20888,7 @@ WC_ASM_ATT_HIDDEN(_sha3_lms_blocksx4_avx2) .p2align 4 _sha3_lms_blocksx4_avx2: #endif /* __APPLE__ */ + movl %r8d, %r8d leaq L_sha3_x4_avx2_r(%rip), %rax movq %rdi, %r9 movq %rdi, %r10 @@ -26444,6 +26448,7 @@ WC_ASM_ATT_HIDDEN(_sha3_lms_chainx4_avx2) .p2align 4 _sha3_lms_chainx4_avx2: #endif /* __APPLE__ */ + movl %ecx, %ecx subq $8, %rsp movl %ecx, %r8d testl %r8d, %r8d @@ -37269,6 +37274,7 @@ _sha3_blocksx4_out_avx2: pushq %r13 pushq %r14 pushq %r15 + movl %edx, %edx leaq L_sha3_x4_avx2_r(%rip), %rax movl %edx, %r12d movq %rsi, %r9 @@ -53575,6 +53581,7 @@ _sha3_blocksx8_out_avx512: pushq %r15 pushq %rbx pushq %rbp + movl %edx, %edx leaq L_sha3_x8_avx512_r(%rip), %rax movl %edx, %r14d movq %rsi, %rcx @@ -59452,6 +59459,7 @@ WC_ASM_ATT_HIDDEN(_sha3_xmss_blocksx8_avx512) .p2align 4 _sha3_xmss_blocksx8_avx512: #endif /* __APPLE__ */ + movl %r8d, %r8d leaq L_sha3_x8_avx512_r(%rip), %rax movl %r8d, %r9d testb $0xff, %r9b @@ -62432,6 +62440,7 @@ WC_ASM_ATT_HIDDEN(_sha3_lms_blocksx8_avx512) .p2align 4 _sha3_lms_blocksx8_avx512: #endif /* __APPLE__ */ + movl %r8d, %r8d leaq L_sha3_x8_avx512_r(%rip), %rax vmovdqu64 (%rdi), %zmm0 vmovdqu64 64(%rdi), %zmm1 @@ -65462,6 +65471,7 @@ WC_ASM_ATT_HIDDEN(_sha3_lms_chainx8_avx512) .p2align 4 _sha3_lms_chainx8_avx512: #endif /* __APPLE__ */ + movl %ecx, %ecx subq $8, %rsp movl %ecx, %r8d testl %r8d, %r8d diff --git a/wolfcrypt/src/sha3_asm.asm b/wolfcrypt/src/sha3_asm.asm index ff1b37fdbf6..1330718416c 100644 --- a/wolfcrypt/src/sha3_asm.asm +++ b/wolfcrypt/src/sha3_asm.asm @@ -4596,6 +4596,7 @@ sha3_block_n_bmi2 PROC push rsi push rbx push rbp + mov r8d, r8d push r9 mov rbp, r9 mov r9, QWORD PTR [rcx] @@ -9530,6 +9531,7 @@ _DATA ENDS _TEXT SEGMENT READONLY PARA sha3_block_n_avx2 PROC push r12 + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -15314,7 +15316,7 @@ _TEXT SEGMENT READONLY PARA sha3_xmss_blocksx4_avx2 PROC push r12 push r13 - mov rax, QWORD PTR [rsp+56] + mov eax, DWORD PTR [rsp+56] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -20809,7 +20811,7 @@ _DATA ENDS _TEXT SEGMENT READONLY PARA sha3_lms_blocksx4_avx2 PROC push r12 - mov rax, QWORD PTR [rsp+48] + mov eax, DWORD PTR [rsp+48] sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -26345,6 +26347,7 @@ _TEXT SEGMENT READONLY PARA sha3_lms_chainx4_avx2 PROC push r12 push r13 + mov r9d, r9d sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -37191,6 +37194,7 @@ sha3_blocksx4_out_avx2 PROC push r15 push rdi push rsi + mov r8d, r8d sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -53526,6 +53530,7 @@ sha3_blocksx8_out_avx512 PROC push rsi push rbx push rbp + mov r8d, r8d sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -59411,7 +59416,7 @@ ptr_L_sha3_xmss_blocksx8_avx512_not_hash QWORD L_sha3_xmss_blocksx8_avx512_not_h _DATA ENDS _TEXT SEGMENT READONLY PARA sha3_xmss_blocksx8_avx512 PROC - mov rax, QWORD PTR [rsp+40] + mov eax, DWORD PTR [rsp+40] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -62366,7 +62371,7 @@ ptr_L_sha3_lms_blocksx8_avx512_j_ff QWORD L_sha3_lms_blocksx8_avx512_j_ff _DATA ENDS _TEXT SEGMENT READONLY PARA sha3_lms_blocksx8_avx512 PROC - mov rax, QWORD PTR [rsp+40] + mov eax, DWORD PTR [rsp+40] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -65371,6 +65376,7 @@ ptr_L_sha3_lms_chainx8_avx512_j_ff QWORD L_sha3_lms_chainx8_avx512_j_ff _DATA ENDS _TEXT SEGMENT READONLY PARA sha3_lms_chainx8_avx512 PROC + mov r9d, r9d sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 diff --git a/wolfcrypt/src/sha512_asm.S b/wolfcrypt/src/sha512_asm.S index 45bd38dfdff..a00d03617f1 100644 --- a/wolfcrypt/src/sha512_asm.S +++ b/wolfcrypt/src/sha512_asm.S @@ -1407,7 +1407,7 @@ _Transform_Sha512_AVX1_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rsi, %rbp + movl %esi, %ebp subq $0x98, %rsp movq 224(%rdi), %rsi leaq L_avx1_sha512_k(%rip), %rdx @@ -3968,7 +3968,7 @@ _Transform_Sha512_AVX1_RORX_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rsi, %rbp + movl %esi, %ebp subq $0x98, %rsp movq 224(%rdi), %rsi leaq L_avx1_rorx_sha512_k(%rip), %rcx @@ -6440,7 +6440,7 @@ _Transform_Sha512_AVX2_Len: pushq %r14 pushq %r15 pushq %rbp - movq %rsi, %rbp + movl %esi, %ebp testb $0x80, %bpl je L_sha512_len_avx2_block movq 224(%rdi), %rbx @@ -9245,6 +9245,7 @@ _Transform_Sha512_AVX2_RORX_Len: pushq %r14 pushq %r15 pushq %rbp + movl %esi, %esi testb $0x80, %sil je L_sha512_len_avx2_rorx_block movq 224(%rdi), %rax diff --git a/wolfcrypt/src/sha512_asm.asm b/wolfcrypt/src/sha512_asm.asm index 732d5268a14..af9ac7bb061 100644 --- a/wolfcrypt/src/sha512_asm.asm +++ b/wolfcrypt/src/sha512_asm.asm @@ -1392,7 +1392,7 @@ Transform_Sha512_AVX1_Len PROC push rsi push rbp mov rdi, rcx - mov rbp, rdx + mov ebp, edx sub rsp, 296 vmovdqu OWORD PTR [rsp+152], xmm6 vmovdqu OWORD PTR [rsp+168], xmm7 @@ -3959,7 +3959,7 @@ Transform_Sha512_AVX1_RORX_Len PROC push rsi push rbp mov rdi, rcx - mov rbp, rdx + mov ebp, edx sub rsp, 296 vmovdqu OWORD PTR [rsp+152], xmm6 vmovdqu OWORD PTR [rsp+168], xmm7 @@ -6425,7 +6425,7 @@ Transform_Sha512_AVX2_Len PROC push rdi push rbp mov rdi, rcx - mov rbp, rdx + mov ebp, edx sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -9224,7 +9224,7 @@ Transform_Sha512_AVX2_RORX_Len PROC push rsi push rbp mov rdi, rcx - mov rsi, rdx + mov esi, edx sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 diff --git a/wolfcrypt/src/wc_frodokem_asm.S b/wolfcrypt/src/wc_frodokem_asm.S index a130abd97fc..c785038ec50 100644 --- a/wolfcrypt/src/wc_frodokem_asm.S +++ b/wolfcrypt/src/wc_frodokem_asm.S @@ -65,6 +65,8 @@ _frodokem_sa_accum_avx2: #endif /* __APPLE__ */ pushq %r12 pushq %r13 + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rcx shlq $1, %rcx addq %rcx, %rsi @@ -128,6 +130,8 @@ _frodokem_as_accum_avx2: pushq %r15 pushq %rbx pushq %rbp + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rcx shlq $4, %rcx addq %rcx, %rdi @@ -243,6 +247,8 @@ _frodokem_mul_bs_avx2: pushq %r15 pushq %rbx pushq %rbp + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rax shlq $1, %rax movq %rdx, %r9 @@ -353,6 +359,8 @@ _frodokem_mul_add_sb_plus_e_avx2: pushq %r14 pushq %r15 pushq %rbx + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rax shlq $1, %rax movq %rdx, %r9 @@ -434,6 +442,7 @@ WC_ASM_ATT_HIDDEN(_frodokem_add_avx2) .p2align 4 _frodokem_add_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vmovd %rdx, %xmm0 vpbroadcastw %xmm0, %ymm0 vmovdqu (%rdi), %ymm1 @@ -471,6 +480,8 @@ WC_ASM_ATT_HIDDEN(_frodokem_a_rows_reduce_avx2) .p2align 4 _frodokem_a_rows_reduce_avx2: #endif /* __APPLE__ */ + movl %esi, %esi + movl %edx, %edx vmovd %rdx, %xmm0 vpbroadcastw %xmm0, %ymm0 movl %esi, %eax @@ -512,6 +523,8 @@ WC_ASM_ATT_HIDDEN(_frodokem_sample_avx2) .p2align 4 _frodokem_sample_avx2: #endif /* __APPLE__ */ + movl %esi, %esi + movl %ecx, %ecx movslq %esi, %rax shlq $1, %rax vpcmpeqw %ymm0, %ymm0, %ymm0 @@ -563,7 +576,10 @@ _frodokem_gen_a_rows_aes_avx2: pushq %r13 pushq %r14 pushq %r15 - movq 40(%rsp), %rax + movl %ecx, %ecx + movl %r8d, %r8d + movl %r9d, %r9d + movl 40(%rsp), %eax vbroadcasti128 (%rdx), %ymm0 vbroadcasti128 16(%rdx), %ymm1 vbroadcasti128 32(%rdx), %ymm2 @@ -718,7 +734,10 @@ _frodokem_gen_a_rows_aes_aesni: pushq %r13 pushq %r14 pushq %r15 - movq 40(%rsp), %rax + movl %ecx, %ecx + movl %r8d, %r8d + movl %r9d, %r9d + movl 40(%rsp), %eax vmovd %rax, %xmm8 vpbroadcastw %xmm8, %xmm8 movslq %r9d, %r10 @@ -915,6 +934,8 @@ _frodokem_sa_accum_avx512: pushq %r15 pushq %rbx pushq %rbp + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rcx shlq $1, %rcx addq %rcx, %rsi @@ -1025,6 +1046,8 @@ _frodokem_as_accum_avx512: pushq %r15 pushq %rbx pushq %rbp + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rcx shlq $4, %rcx addq %rcx, %rdi @@ -1186,6 +1209,8 @@ _frodokem_mul_bs_avx512: pushq %r15 pushq %rbx pushq %rbp + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rax shlq $1, %rax movq %rax, %rcx @@ -1342,6 +1367,8 @@ _frodokem_mul_add_sb_plus_e_avx512: pushq %r14 pushq %r15 pushq %rbx + movl %ecx, %ecx + movl %r8d, %r8d movslq %ecx, %rax shlq $1, %rax movq %rdx, %r9 @@ -1415,6 +1442,7 @@ WC_ASM_ATT_HIDDEN(_frodokem_add_avx512) .p2align 4 _frodokem_add_avx512: #endif /* __APPLE__ */ + movl %edx, %edx vmovd %rdx, %xmm0 vpbroadcastw %xmm0, %zmm0 vmovdqu64 (%rdi), %zmm1 @@ -1444,6 +1472,8 @@ WC_ASM_ATT_HIDDEN(_frodokem_sample_avx512) .p2align 4 _frodokem_sample_avx512: #endif /* __APPLE__ */ + movl %esi, %esi + movl %ecx, %ecx movslq %esi, %rax shlq $1, %rax vpternlogd $0xff, %zmm0, %zmm0, %zmm0 @@ -1495,7 +1525,10 @@ _frodokem_gen_a_rows_aes_avx512: pushq %r13 pushq %r14 pushq %r15 - movq 40(%rsp), %rax + movl %ecx, %ecx + movl %r8d, %r8d + movl %r9d, %r9d + movl 40(%rsp), %eax vbroadcasti32x4 (%rdx), %zmm0 vbroadcasti32x4 16(%rdx), %zmm1 vbroadcasti32x4 32(%rdx), %zmm2 diff --git a/wolfcrypt/src/wc_frodokem_asm.asm b/wolfcrypt/src/wc_frodokem_asm.asm index 815c83b6247..f6be6c818a2 100644 --- a/wolfcrypt/src/wc_frodokem_asm.asm +++ b/wolfcrypt/src/wc_frodokem_asm.asm @@ -61,7 +61,8 @@ frodokem_sa_accum_avx2 PROC push r13 push r14 push r15 - mov rax, QWORD PTR [rsp+72] + mov r9d, r9d + mov eax, DWORD PTR [rsp+72] movsxd r9, r9d shl r9, 1 add rdx, r9 @@ -116,7 +117,8 @@ frodokem_as_accum_avx2 PROC push rsi push rbx push rbp - mov rax, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+104] sub rsp, 120 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -239,7 +241,8 @@ frodokem_mul_bs_avx2 PROC push rsi push rbx push rbp - mov rax, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+104] sub rsp, 120 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -357,7 +360,8 @@ frodokem_mul_add_sb_plus_e_avx2 PROC push rdi push rsi push rbx - mov rax, QWORD PTR [rsp+96] + mov r9d, r9d + mov eax, DWORD PTR [rsp+96] sub rsp, 32 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -434,6 +438,7 @@ frodokem_mul_add_sb_plus_e_avx2 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA frodokem_add_avx2 PROC + mov r8d, r8d vmovd xmm0, r8 vpbroadcastw ymm0, xmm0 vmovdqu ymm1, YMMWORD PTR [rcx] @@ -458,6 +463,8 @@ frodokem_add_avx2 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA frodokem_a_rows_reduce_avx2 PROC + mov edx, edx + mov r8d, r8d vmovd xmm0, r8 vpbroadcastw ymm0, xmm0 mov eax, edx @@ -487,6 +494,8 @@ _TEXT ENDS _TEXT SEGMENT READONLY PARA frodokem_sample_avx2 PROC push r12 + mov edx, edx + mov r9d, r9d sub rsp, 32 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -534,9 +543,10 @@ frodokem_gen_a_rows_aes_avx2 PROC push r15 push rdi push rsi - mov rax, QWORD PTR [rsp+88] - mov r10, QWORD PTR [rsp+96] - mov r11, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+88] + mov r10d, DWORD PTR [rsp+96] + mov r11d, DWORD PTR [rsp+104] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -704,9 +714,10 @@ frodokem_gen_a_rows_aes_aesni PROC push r15 push rdi push rsi - mov rax, QWORD PTR [rsp+88] - mov r10, QWORD PTR [rsp+96] - mov r11, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+88] + mov r10d, DWORD PTR [rsp+96] + mov r11d, DWORD PTR [rsp+104] sub rsp, 56 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -909,7 +920,8 @@ frodokem_sa_accum_avx512 PROC push rsi push rbx push rbp - mov rax, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+104] sub rsp, 72 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -1021,7 +1033,8 @@ frodokem_as_accum_avx512 PROC push rsi push rbx push rbp - mov rax, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+104] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -1196,7 +1209,8 @@ frodokem_mul_bs_avx512 PROC push rsi push rbx push rbp - mov rax, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+104] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -1366,7 +1380,8 @@ frodokem_mul_add_sb_plus_e_avx512 PROC push rdi push rsi push rbx - mov rax, QWORD PTR [rsp+96] + mov r9d, r9d + mov eax, DWORD PTR [rsp+96] movsxd r10, r9d shl r10, 1 mov r11, r8 @@ -1429,6 +1444,7 @@ frodokem_mul_add_sb_plus_e_avx512 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA frodokem_add_avx512 PROC + mov r8d, r8d vmovd xmm0, r8 vpbroadcastw zmm0, xmm0 vmovdqu64 zmm1, [rcx] @@ -1446,6 +1462,8 @@ _TEXT ENDS _TEXT SEGMENT READONLY PARA frodokem_sample_avx512 PROC push r12 + mov edx, edx + mov r9d, r9d sub rsp, 32 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -1493,9 +1511,10 @@ frodokem_gen_a_rows_aes_avx512 PROC push r15 push rdi push rsi - mov rax, QWORD PTR [rsp+88] - mov r10, QWORD PTR [rsp+96] - mov r11, QWORD PTR [rsp+104] + mov r9d, r9d + mov eax, DWORD PTR [rsp+88] + mov r10d, DWORD PTR [rsp+96] + mov r11d, DWORD PTR [rsp+104] sub rsp, 168 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 diff --git a/wolfcrypt/src/wc_mldsa_asm.S b/wolfcrypt/src/wc_mldsa_asm.S index 89e71b21a54..3ddfb64f056 100644 --- a/wolfcrypt/src/wc_mldsa_asm.S +++ b/wolfcrypt/src/wc_mldsa_asm.S @@ -21028,7 +21028,8 @@ _wc_mldsa_rej_uniform_n_avx2: pushq %r12 pushq %r13 pushq %r14 - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax vmovdqu L_mldsa_rej_q(%rip), %ymm6 vmovdqu L_mldsa_rej_mask(%rip), %ymm7 @@ -21568,7 +21569,8 @@ _wc_mldsa_rej_uniform_avx2: pushq %r12 pushq %r13 pushq %r14 - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax cmpl $0x00, %esi je L_mldsa_rej_uniform_avx2_done_64 @@ -21961,6 +21963,7 @@ _wc_mldsa_extract_coeffs_eta2_avx2: pushq %rbx pushq %r12 pushq %r13 + movl %esi, %esi vmovdqu L_mldsa_extract_coeffs_eta2_mask_nibbles(%rip), %ymm6 vmovdqu L_mldsa_extract_coeffs_eta2_mul(%rip), %ymm7 vmovdqu L_mldsa_extract_coeffs_eta2_five(%rip), %ymm8 @@ -22601,6 +22604,7 @@ _wc_mldsa_extract_coeffs_eta4_avx2: pushq %rbx pushq %r12 pushq %r13 + movl %esi, %esi vmovdqu L_mldsa_extract_coeffs_eta4_mask_nibbles(%rip), %ymm6 vmovdqu L_mldsa_extract_coeffs_eta4_nine(%rip), %ymm7 vmovdqu L_mldsa_extract_coeffs_eta4_four(%rip), %ymm8 @@ -39064,6 +39068,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_check_low_avx2) .p2align 4 _wc_mldsa_vec_check_low_avx2: #endif /* __APPLE__ */ + movl %edx, %edx subl $0x01, %edx movd %edx, %xmm2 negl %edx @@ -39854,7 +39859,8 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_rej_uniform_n_avx512) _wc_mldsa_rej_uniform_n_avx512: #endif /* __APPLE__ */ pushq %rbx - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax vbroadcasti64x4 L_mldsa_rej_q_avx512(%rip), %zmm1 vbroadcasti64x4 L_mldsa_rej_mask_avx512(%rip), %zmm2 @@ -39932,7 +39938,8 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_rej_uniform_avx512) _wc_mldsa_rej_uniform_avx512: #endif /* __APPLE__ */ pushq %rbx - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax vbroadcasti64x4 L_mldsa_rej_q_avx512(%rip), %zmm1 vbroadcasti64x4 L_mldsa_rej_mask_avx512(%rip), %zmm2 @@ -40085,6 +40092,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_extract_coeffs_eta2_avx512) _wc_mldsa_extract_coeffs_eta2_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi vmovdqu64 L_mldsa_extract_coeffs_eta2_avx512_dup(%rip), %zmm2 vmovdqu64 L_mldsa_extract_coeffs_eta2_avx512_shift(%rip), %zmm3 vmovdqu64 L_mldsa_extract_coeffs_eta2_avx512_mask_nibbles(%rip), %zmm4 @@ -40424,6 +40432,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_extract_coeffs_eta4_avx512) _wc_mldsa_extract_coeffs_eta4_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi vmovdqu64 L_mldsa_extract_coeffs_eta4_avx512_dup(%rip), %zmm2 vmovdqu64 L_mldsa_extract_coeffs_eta4_avx512_shift(%rip), %zmm3 vmovdqu64 L_mldsa_extract_coeffs_eta4_avx512_mask_nibbles(%rip), %zmm4 @@ -40748,6 +40757,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_make_hint_88_avx512) _wc_mldsa_make_hint_88_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %edx, %edx vmovdqu64 L_mldsa_make_hint_88_avx512_low(%rip), %zmm3 vmovdqu64 L_mldsa_make_hint_88_avx512_neg_low(%rip), %zmm4 vmovdqu64 L_mldsa_make_hint_88_avx512_seq(%rip), %zmm1 @@ -41178,6 +41188,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_make_hint_32_avx512) _wc_mldsa_make_hint_32_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %edx, %edx vmovdqu64 L_mldsa_make_hint_32_avx512_low(%rip), %zmm3 vmovdqu64 L_mldsa_make_hint_32_avx512_neg_low(%rip), %zmm4 vmovdqu64 L_mldsa_make_hint_32_avx512_seq(%rip), %zmm1 @@ -45311,6 +45322,7 @@ _wc_mldsa_redistribute_21_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -45511,6 +45523,7 @@ _wc_mldsa_redistribute_17_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -63455,6 +63468,7 @@ WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_check_low_avx512) .p2align 4 _wc_mldsa_vec_check_low_avx512: #endif /* __APPLE__ */ + movl %edx, %edx subl $0x01, %edx movd %edx, %xmm2 negl %edx diff --git a/wolfcrypt/src/wc_mldsa_asm.asm b/wolfcrypt/src/wc_mldsa_asm.asm index 9649525a62c..ff8619c5cf0 100644 --- a/wolfcrypt/src/wc_mldsa_asm.asm +++ b/wolfcrypt/src/wc_mldsa_asm.asm @@ -20992,9 +20992,9 @@ wc_mldsa_rej_uniform_n_avx2 PROC push r14 push r15 push rdi - mov r10, r9 + mov r10d, r9d mov r9, r8 - mov r8, rdx + mov r8d, edx mov rdx, rcx sub rsp, 72 vmovdqu OWORD PTR [rsp+8], xmm6 @@ -21536,9 +21536,9 @@ wc_mldsa_rej_uniform_avx2 PROC push r14 push r15 push rdi - mov r10, r9 + mov r10d, r9d mov r9, r8 - mov r8, rdx + mov r8d, edx mov rdx, rcx mov eax, r8d cmp r8d, 0 @@ -21881,6 +21881,7 @@ wc_mldsa_extract_coeffs_eta2_avx2 PROC push r13 push r14 push r15 + mov edx, edx sub rsp, 64 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -22501,6 +22502,7 @@ wc_mldsa_extract_coeffs_eta4_avx2 PROC push r13 push r14 push r15 + mov edx, edx sub rsp, 48 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -38317,6 +38319,7 @@ wc_mldsa_use_hint_32_avx2 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA wc_mldsa_vec_check_low_avx2 PROC + mov r8d, r8d sub rsp, 56 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -39069,9 +39072,9 @@ _TEXT SEGMENT READONLY PARA wc_mldsa_rej_uniform_n_avx512 PROC push rbx push r12 - mov r10, r9 + mov r10d, r9d mov r9, r8 - mov r8, rdx + mov r8d, edx mov rdx, rcx mov eax, r8d vbroadcasti64x4 zmm1, YMMWORD PTR L_mldsa_rej_q_avx512 @@ -39139,9 +39142,9 @@ _TEXT SEGMENT READONLY PARA wc_mldsa_rej_uniform_avx512 PROC push rbx push r12 - mov r10, r9 + mov r10d, r9d mov r9, r8 - mov r8, rdx + mov r8d, edx mov rdx, rcx mov eax, r8d vbroadcasti64x4 zmm1, YMMWORD PTR L_mldsa_rej_q_avx512 @@ -39250,6 +39253,7 @@ wc_mldsa_extract_coeffs_eta2_avx512 PROC push rbx push r12 push r13 + mov edx, edx sub rsp, 16 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu64 zmm2, ZMMWORD PTR L_mldsa_extract_coeffs_eta2_avx512_dup @@ -39549,6 +39553,7 @@ wc_mldsa_extract_coeffs_eta4_avx512 PROC push rbx push r12 push r13 + mov edx, edx sub rsp, 16 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu64 zmm2, ZMMWORD PTR L_mldsa_extract_coeffs_eta4_avx512_dup @@ -39838,6 +39843,7 @@ _DATA ENDS _TEXT SEGMENT READONLY PARA wc_mldsa_make_hint_88_avx512 PROC push rbx + mov r8d, r8d mov r10, QWORD PTR [rsp+48] sub rsp, 16 vmovdqu OWORD PTR [rsp], xmm6 @@ -40232,6 +40238,7 @@ _DATA ENDS _TEXT SEGMENT READONLY PARA wc_mldsa_make_hint_32_avx512 PROC push rbx + mov r8d, r8d mov r10, QWORD PTR [rsp+48] sub rsp, 16 vmovdqu OWORD PTR [rsp], xmm6 @@ -44361,6 +44368,7 @@ wc_mldsa_redistribute_21_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -44574,6 +44582,7 @@ wc_mldsa_redistribute_17_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -62054,6 +62063,7 @@ wc_mldsa_decompose_q32_avx512 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA wc_mldsa_vec_check_low_avx512 PROC + mov r8d, r8d sub r8d, 1 movd xmm2, r8d neg r8d diff --git a/wolfcrypt/src/wc_mlkem_asm.S b/wolfcrypt/src/wc_mlkem_asm.S index 897c6c54ad0..1e543dce3ef 100644 --- a/wolfcrypt/src/wc_mlkem_asm.S +++ b/wolfcrypt/src/wc_mlkem_asm.S @@ -511,6 +511,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_keygen_avx2) _mlkem_keygen_avx2: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d vmovdqu mlkem_q(%rip), %ymm14 vmovdqu mlkem_v(%rip), %ymm15 movq %rdi, %r11 @@ -3178,7 +3179,7 @@ _mlkem_encapsulate_avx2: pushq %rbx movq 48(%rsp), %rax movq 56(%rsp), %r10 - movq 64(%rsp), %r11 + movl 64(%rsp), %r11d subq $48, %rsp vmovdqu mlkem_q(%rip), %ymm14 vmovdqu mlkem_v(%rip), %ymm15 @@ -7983,6 +7984,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decapsulate_avx2) .p2align 4 _mlkem_decapsulate_avx2: #endif /* __APPLE__ */ + movl %r8d, %r8d vmovdqu mlkem_q(%rip), %ymm14 vmovdqu mlkem_v(%rip), %ymm15 movslq %r8d, %rax @@ -11047,7 +11049,8 @@ _mlkem_rej_uniform_n_avx2: pushq %r14 pushq %r15 pushq %rbp - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax vmovdqu L_mlkem_rej_q(%rip), %ymm6 vmovdqu L_mlkem_rej_ones(%rip), %ymm7 @@ -11664,7 +11667,8 @@ _mlkem_rej_uniform_avx2: pushq %r14 pushq %r15 pushq %rbp - movq %rcx, %r8 + movl %esi, %esi + movl %ecx, %r8d movl %esi, %eax cmpl $0x00, %esi je L_mlkem_rej_uniform_avx2_done_64 @@ -12601,6 +12605,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_compress_10_avx2) .p2align 4 _mlkem_compress_10_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu (%rsi), %ymm0 vmovdqu L_mlkem_compress_10_avx2_mask(%rip), %ymm9 vmovdqu L_mlkem_compress_10_avx2_shift(%rip), %ymm8 @@ -12992,6 +12997,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_10_avx2) .p2align 4 _mlkem_decompress_10_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu L_mlkem_decompress_10_avx2_mask(%rip), %ymm4 vmovdqu L_mlkem_decompress_10_avx2_q(%rip), %ymm5 vmovdqu L_mlkem_decompress_10_avx2_shuf(%rip), %ymm6 @@ -13238,6 +13244,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_compress_11_avx2) .p2align 4 _mlkem_compress_11_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu (%rsi), %ymm0 vmovdqu L_mlkem_compress_11_avx2_v(%rip), %ymm7 vmovdqu L_mlkem_compress_11_avx2_off(%rip), %ymm8 @@ -13704,6 +13711,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_11_avx2) .p2align 4 _mlkem_decompress_11_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu L_mlkem_decompress_11_avx2_q(%rip), %ymm4 vmovdqu L_mlkem_decompress_11_avx2_shuf(%rip), %ymm5 vmovdqu L_mlkem_decompress_11_avx2_sllv(%rip), %ymm6 @@ -15425,6 +15433,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_cmp_avx2) .p2align 4 _mlkem_cmp_avx2: #endif /* __APPLE__ */ + movl %edx, %edx vpxor %ymm2, %ymm2, %ymm2 vpxor %ymm3, %ymm3, %ymm3 movl $0x00, %ecx @@ -16364,6 +16373,7 @@ _mlkem_keygen_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %r8d, %r8d subq $0x400, %rsp movq $0xd01, %r14 vmovd %r14d, %xmm14 @@ -20024,7 +20034,7 @@ _mlkem_encapsulate_avx512: pushq %rbp movq 56(%rsp), %rax movq 64(%rsp), %r10 - movq 72(%rsp), %r11 + movl 72(%rsp), %r11d subq $56, %rsp movq $0xd01, %r13 vmovd %r13d, %xmm14 @@ -26789,6 +26799,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decapsulate_avx512) _mlkem_decapsulate_avx512: #endif /* __APPLE__ */ pushq %r12 + movl %r8d, %r8d movq $0xd01, %r12 vmovd %r12d, %xmm14 vpbroadcastw %xmm14, %zmm14 @@ -30237,6 +30248,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_ptacc_x2_avx512) .p2align 4 _mlkem_ptacc_x2_avx512: #endif /* __APPLE__ */ + movl %ecx, %ecx movq $0xd01, %r9 vmovd %r9d, %xmm12 vpbroadcastw %xmm12, %zmm12 @@ -31432,6 +31444,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_avx512) _mlkem_rej_uniform_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -31569,6 +31583,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_n_avx512) _mlkem_rej_uniform_n_avx512: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -32201,6 +32217,7 @@ _mlkem_redistribute_8_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -32281,6 +32298,7 @@ _mlkem_redistribute_16_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -32401,6 +32419,7 @@ _mlkem_redistribute_17_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -32537,6 +32556,7 @@ _mlkem_redistribute_21_rand_x8_avx512: pushq %r12 pushq %r13 pushq %r14 + movl %edx, %edx movl %edx, %r13d movq %rsi, %rax addq %r13, %rax @@ -32804,6 +32824,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_cmp_avx512) .p2align 4 _mlkem_cmp_avx512: #endif /* __APPLE__ */ + movl %edx, %edx vpxorq %zmm0, %zmm0, %zmm0 vpxor %ymm2, %ymm2, %ymm2 movl $0x00, %ecx @@ -33858,6 +33879,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_10_avx512) .p2align 4 _mlkem_decompress_10_avx512: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu64 L_mlkem_decompress_10_avx512_mask(%rip), %zmm0 vmovdqu64 L_mlkem_decompress_10_avx512_q(%rip), %zmm1 vmovdqu64 L_mlkem_decompress_10_avx512_shuf(%rip), %zmm2 @@ -34052,6 +34074,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_11_avx512) .p2align 4 _mlkem_decompress_11_avx512: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu64 L_mlkem_decompress_11_avx512_q(%rip), %zmm0 vmovdqu64 L_mlkem_decompress_11_avx512_shuf(%rip), %zmm1 vmovdqu64 L_mlkem_decompress_11_avx512_srlvd(%rip), %zmm2 @@ -34409,6 +34432,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_compress_10_avx512) .p2align 4 _mlkem_compress_10_avx512: #endif /* __APPLE__ */ + movl %edx, %edx movq $0x4ebf, %rax vmovd %eax, %xmm0 vpbroadcastw %xmm0, %zmm0 @@ -34727,6 +34751,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_compress_11_avx512) .p2align 4 _mlkem_compress_11_avx512: #endif /* __APPLE__ */ + movl %edx, %edx movq $0x4ebf, %rax vmovd %eax, %xmm0 vpbroadcastw %xmm0, %zmm0 @@ -35092,6 +35117,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_avx512_vbmi2) _mlkem_rej_uniform_avx512_vbmi2: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -35217,6 +35244,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_n_avx512_vbmi2) _mlkem_rej_uniform_n_avx512_vbmi2: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -35624,6 +35653,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_avx512_vbmi) _mlkem_rej_uniform_avx512_vbmi: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -35760,6 +35791,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_n_avx512_vbmi) _mlkem_rej_uniform_n_avx512_vbmi: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -36185,6 +36218,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_compress_10_avx512_vbmi) .p2align 4 _mlkem_compress_10_avx512_vbmi: #endif /* __APPLE__ */ + movl %edx, %edx movq $0x4ebf, %rax vmovd %eax, %xmm0 vpbroadcastw %xmm0, %zmm0 @@ -36452,6 +36486,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_10_avx512_vbmi) .p2align 4 _mlkem_decompress_10_avx512_vbmi: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu64 L_mlkem_decompress_10_avx512_vbmi_mask(%rip), %zmm0 vmovdqu64 L_mlkem_decompress_10_avx512_vbmi_q(%rip), %zmm1 vmovdqu64 L_mlkem_decompress_10_avx512_vbmi_sllv(%rip), %zmm2 @@ -36642,6 +36677,7 @@ WC_ASM_ATT_HIDDEN(_mlkem_decompress_11_avx512_vbmi) .p2align 4 _mlkem_decompress_11_avx512_vbmi: #endif /* __APPLE__ */ + movl %edx, %edx vmovdqu64 L_mlkem_decompress_11_avx512_vbmi_q(%rip), %zmm0 vmovdqu64 L_mlkem_decompress_11_avx512_vbmi_srlvd(%rip), %zmm1 vmovdqu64 L_mlkem_decompress_11_avx512_vbmi_srlvq(%rip), %zmm2 @@ -36788,6 +36824,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_avx512_vbmi_vbmi2) _mlkem_rej_uniform_avx512_vbmi_vbmi2: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 @@ -36912,6 +36950,8 @@ WC_ASM_ATT_HIDDEN(_mlkem_rej_uniform_n_avx512_vbmi_vbmi2) _mlkem_rej_uniform_n_avx512_vbmi_vbmi2: #endif /* __APPLE__ */ pushq %rbx + movl %esi, %esi + movl %ecx, %ecx movl %esi, %eax movq $0xd01, %r10 vmovd %r10d, %xmm0 diff --git a/wolfcrypt/src/wc_mlkem_asm.asm b/wolfcrypt/src/wc_mlkem_asm.asm index 43db647d3fb..8f22f0098b4 100644 --- a/wolfcrypt/src/wc_mlkem_asm.asm +++ b/wolfcrypt/src/wc_mlkem_asm.asm @@ -450,7 +450,7 @@ mlkem_keygen_avx2 PROC push r12 push r13 push r14 - mov rax, QWORD PTR [rsp+64] + mov eax, DWORD PTR [rsp+64] sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -3133,7 +3133,7 @@ mlkem_encapsulate_avx2 PROC mov r10, QWORD PTR [rsp+104] mov r11, QWORD PTR [rsp+112] mov r12, QWORD PTR [rsp+120] - mov r13, QWORD PTR [rsp+128] + mov r13d, DWORD PTR [rsp+128] sub rsp, 208 vmovdqu OWORD PTR [rsp+48], xmm6 vmovdqu OWORD PTR [rsp+64], xmm7 @@ -7948,7 +7948,7 @@ _TEXT ENDS _TEXT SEGMENT READONLY PARA mlkem_decapsulate_avx2 PROC push r12 - mov rax, QWORD PTR [rsp+48] + mov eax, DWORD PTR [rsp+48] sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -10993,6 +10993,8 @@ mlkem_rej_uniform_n_avx2 PROC push rdi push rsi push rbp + mov edx, edx + mov r9d, r9d mov r10, rcx sub rsp, 72 vmovdqu OWORD PTR [rsp+8], xmm6 @@ -11611,6 +11613,8 @@ mlkem_rej_uniform_avx2 PROC push rdi push rsi push rbp + mov edx, edx + mov r9d, r9d mov r10, rcx sub rsp, 72 vmovdqu OWORD PTR [rsp+8], xmm6 @@ -12434,6 +12438,7 @@ ptr_L_mlkem_compress_10_avx2_shift12 QWORD L_mlkem_compress_10_avx2_shift12 _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_compress_10_avx2 PROC + mov r8d, r8d sub rsp, 136 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -12802,6 +12807,7 @@ ptr_L_mlkem_decompress_10_avx2_shuf QWORD L_mlkem_decompress_10_avx2_shuf _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_10_avx2 PROC + mov r8d, r8d sub rsp, 40 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -12985,6 +12991,7 @@ ptr_L_mlkem_compress_11_avx2_shuf QWORD L_mlkem_compress_11_avx2_shuf _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_compress_11_avx2 PROC + mov r8d, r8d sub rsp, 152 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -13416,6 +13423,7 @@ ptr_L_mlkem_decompress_11_avx2_mask QWORD L_mlkem_decompress_11_avx2_mask _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_11_avx2 PROC + mov r8d, r8d sub rsp, 72 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -14914,6 +14922,7 @@ mlkem_to_bytes_avx2 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA mlkem_cmp_avx2 PROC + mov r8d, r8d vpxor ymm2, ymm2, ymm2 vpxor ymm3, ymm3, ymm3 mov r9d, 0 @@ -15846,7 +15855,7 @@ mlkem_keygen_avx512 PROC push r14 push r15 push rdi - mov rax, QWORD PTR [rsp+80] + mov eax, DWORD PTR [rsp+80] sub rsp, 1184 vmovdqu OWORD PTR [rsp+1024], xmm6 vmovdqu OWORD PTR [rsp+1040], xmm7 @@ -19520,7 +19529,7 @@ mlkem_encapsulate_avx512 PROC mov r10, QWORD PTR [rsp+112] mov r11, QWORD PTR [rsp+120] mov r12, QWORD PTR [rsp+128] - mov r13, QWORD PTR [rsp+136] + mov r13d, DWORD PTR [rsp+136] sub rsp, 216 vmovdqu OWORD PTR [rsp+56], xmm6 vmovdqu OWORD PTR [rsp+72], xmm7 @@ -26296,7 +26305,7 @@ mlkem_decapsulate_avx512 PROC push r12 push r13 push r14 - mov rax, QWORD PTR [rsp+64] + mov eax, DWORD PTR [rsp+64] sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -29755,6 +29764,7 @@ _TEXT ENDS _TEXT SEGMENT READONLY PARA mlkem_ptacc_x2_avx512 PROC push r12 + mov r9d, r9d sub rsp, 128 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -30873,6 +30883,8 @@ mlkem_rej_uniform_avx512 PROC push rbx push r12 push r13 + mov edx, edx + mov r9d, r9d sub rsp, 48 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -30988,6 +31000,8 @@ mlkem_rej_uniform_n_avx512 PROC push rbx push r12 push r13 + mov edx, edx + mov r9d, r9d sub rsp, 48 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -31475,6 +31489,7 @@ mlkem_redistribute_8_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -31568,6 +31583,7 @@ mlkem_redistribute_16_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -31701,6 +31717,7 @@ mlkem_redistribute_17_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -31850,6 +31867,7 @@ mlkem_redistribute_21_rand_x8_avx512 PROC push r14 push r15 push rdi + mov r8d, r8d sub rsp, 160 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -32131,6 +32149,7 @@ mlkem_csubq_avx512 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA mlkem_cmp_avx512 PROC + mov r8d, r8d vpxorq zmm0, zmm0, zmm0 vpxor ymm2, ymm2, ymm2 mov r9d, 0 @@ -32937,6 +32956,7 @@ ptr_L_mlkem_decompress_10_avx512_shuf QWORD L_mlkem_decompress_10_avx512_shuf _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_10_avx512 PROC + mov r8d, r8d vmovdqu64 zmm0, ZMMWORD PTR L_mlkem_decompress_10_avx512_mask vmovdqu64 zmm1, ZMMWORD PTR L_mlkem_decompress_10_avx512_q vmovdqu64 zmm2, ZMMWORD PTR L_mlkem_decompress_10_avx512_shuf @@ -33076,6 +33096,7 @@ ptr_L_mlkem_decompress_11_avx512_mask QWORD L_mlkem_decompress_11_avx512_mask _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_11_avx512 PROC + mov r8d, r8d sub rsp, 40 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -33374,6 +33395,7 @@ ptr_L_mlkem_compress_10_avx512_shuf QWORD L_mlkem_compress_10_avx512_shuf _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_compress_10_avx512 PROC + mov r8d, r8d sub rsp, 120 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -33667,6 +33689,7 @@ ptr_L_mlkem_compress_11_avx512_shuf QWORD L_mlkem_compress_11_avx512_shuf _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_compress_11_avx512 PROC + mov r8d, r8d sub rsp, 136 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -34017,6 +34040,8 @@ _TEXT SEGMENT READONLY PARA mlkem_rej_uniform_avx512_vbmi2 PROC push rbx push r12 + mov edx, edx + mov r9d, r9d mov eax, edx mov r12, 3329 vmovd xmm0, r12d @@ -34110,6 +34135,8 @@ _TEXT SEGMENT READONLY PARA mlkem_rej_uniform_n_avx512_vbmi2 PROC push rbx push r12 + mov edx, edx + mov r9d, r9d mov eax, edx mov r12, 3329 vmovd xmm0, r12d @@ -34429,6 +34456,8 @@ mlkem_rej_uniform_avx512_vbmi PROC push rbx push r12 push r13 + mov edx, edx + mov r9d, r9d sub rsp, 48 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -34543,6 +34572,8 @@ mlkem_rej_uniform_n_avx512_vbmi PROC push rbx push r12 push r13 + mov edx, edx + mov r9d, r9d sub rsp, 48 vmovdqu OWORD PTR [rsp], xmm6 vmovdqu OWORD PTR [rsp+16], xmm7 @@ -34897,6 +34928,7 @@ ptr_L_mlkem_compress_10_avx512_vbmi_gather QWORD L_mlkem_compress_10_avx512_vbmi _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_compress_10_avx512_vbmi PROC + mov r8d, r8d sub rsp, 120 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -35132,6 +35164,7 @@ ptr_L_mlkem_decompress_10_avx512_vbmi_hi QWORD L_mlkem_decompress_10_avx512_vbmi _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_10_avx512_vbmi PROC + mov r8d, r8d vmovdqu64 zmm0, ZMMWORD PTR L_mlkem_decompress_10_avx512_vbmi_mask vmovdqu64 zmm1, ZMMWORD PTR L_mlkem_decompress_10_avx512_vbmi_q vmovdqu64 zmm2, ZMMWORD PTR L_mlkem_decompress_10_avx512_vbmi_sllv @@ -35260,6 +35293,7 @@ ptr_L_mlkem_decompress_11_avx512_vbmi_hi QWORD L_mlkem_decompress_11_avx512_vbmi _DATA ENDS _TEXT SEGMENT READONLY PARA mlkem_decompress_11_avx512_vbmi PROC + mov r8d, r8d sub rsp, 40 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 @@ -35379,6 +35413,8 @@ _TEXT SEGMENT READONLY PARA mlkem_rej_uniform_avx512_vbmi_vbmi2 PROC push rbx push r12 + mov edx, edx + mov r9d, r9d mov eax, edx mov r12, 3329 vmovd xmm0, r12d @@ -35471,6 +35507,8 @@ _TEXT SEGMENT READONLY PARA mlkem_rej_uniform_n_avx512_vbmi_vbmi2 PROC push rbx push r12 + mov edx, edx + mov r9d, r9d mov eax, edx mov r12, 3329 vmovd xmm0, r12d diff --git a/wolfssl/wolfcrypt/cpuid.h b/wolfssl/wolfcrypt/cpuid.h index 4d7159d02eb..748b28f342d 100644 --- a/wolfssl/wolfcrypt/cpuid.h +++ b/wolfssl/wolfcrypt/cpuid.h @@ -106,6 +106,19 @@ typedef word32 cpuid_flags_t; * (vpshufb, vpaddw, vpmulhw, vpackusdw, ... on zmm). */ #define CPUID_AVX512_BW 0x80000 #define CPUID_SSSE3 0x100000 /* SSSE3 (pshufb etc.) */ + /* Tuning bits. These are not instruction-set features: they mark a CPU + * on which an implementation that the feature bits allow is measured to + * be slower than a narrower one. Set from the family/model in cpuid.c + * (none are set when WC_CPUID_NO_TUNING is defined) and can be changed + * with cpuid_set_flag()/cpuid_clear_flag(). + * CPUID_SLOW_VAES: AES using VAES (ymm) is no faster than the + * AVX2/AES-NI code, so AES (and AES-GCM-SIV's + * POLYVAL) does not pick it. + * CPUID_SLOW_AVX2_SHA3: the single-stream AVX2 Keccak-f[1600] block is + * slower than BMI2 and C, so SHA-3 and ML-DSA do + * not pick it. */ + #define CPUID_SLOW_VAES 0x200000 + #define CPUID_SLOW_AVX2_SHA3 0x400000 #define IS_INTEL_AVX1(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AVX1) #define IS_INTEL_AVX2(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AVX2) @@ -146,6 +159,10 @@ typedef word32 cpuid_flags_t; #define IS_CPU_INTEL(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_INTEL) #define IS_CPU_AMD(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AMD) #define IS_INTEL_SSSE3(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SSSE3) + #define IS_CPU_SLOW_VAES(f) \ + (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SLOW_VAES) + #define IS_CPU_SLOW_AVX2_SHA3(f) \ + (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SLOW_AVX2_SHA3) #elif defined(HAVE_CPUID_AARCH64) diff --git a/wolfssl/wolfcrypt/sha3.h b/wolfssl/wolfcrypt/sha3.h index 9aaa86502dc..da56771723a 100644 --- a/wolfssl/wolfcrypt/sha3.h +++ b/wolfssl/wolfcrypt/sha3.h @@ -408,8 +408,11 @@ WOLFSSL_LOCAL void BlockSha3(word64 *s); * functions. Measured (Ethereum "Optimizing Keccak"; OpenSSL * keccak1600-x86_64.pl): AVX2 is ~13-17% faster than BMI2 on Intel * Haswell..Skylake, tied on Ice Lake, but ~2x SLOWER on AMD Zen, so AVX2 - * is Intel-only. (Single-stream AVX-512 is vpermt2q-bound and slower - * than BMI2 everywhere measured, so it is not built.) + * is Intel-only. On Intel Arrow Lake and Lunar Lake AVX2 is slower + * than BMI2 and C, so it is also skipped on CPUs marked + * CPUID_SLOW_AVX2_SHA3. + * (Single-stream AVX-512 is vpermt2q-bound and slower than BMI2 + * everywhere measured, so it is not built.) * Every caller of sha3_block_avx2()/sha3_block_n_avx2() must select with * this and not with IS_INTEL_AVX2() alone. * sha3.c puts BMI2 ahead of AVX2 because the BMI2 block needs no @@ -421,7 +424,8 @@ WOLFSSL_LOCAL void BlockSha3(word64 *s); #elif defined(WOLFSSL_SHA3_AVX2) #define SHA3_USE_AVX2(f) IS_INTEL_AVX2(f) #else - #define SHA3_USE_AVX2(f) (IS_INTEL_AVX2(f) && IS_CPU_INTEL(f)) + #define SHA3_USE_AVX2(f) \ + (IS_INTEL_AVX2(f) && IS_CPU_INTEL(f) && !IS_CPU_SLOW_AVX2_SHA3(f)) #endif WOLFSSL_LOCAL void sha3_block_n_bmi2(word64* s, const byte* data, word32 n,