New 3-way batched Keccak-f1600 (x3) improves ML-DSA performance by 2.5 to 13.0% on Neoverse V3

.github/actions/config-variations/action.yml

@@ -8,7 +8,7 @@ inputs:
     description: 'GitHub token'
     required: true
   tests:
-    description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized'
+    description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, x3-keccak, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized'
     required: false
     default: 'all'
   opt:
@@ -236,6 +236,40 @@ runs:
         alloc: false # Requires custom config
         sign_hook: false # Requires custom config
         rng_fail: true
+    # On AArch64, build with SHA3 so that the native x3 hybrid kernel is used;
+    # elsewhere, this tests the C fallback of the 3-way batched Keccak path.
+    - name: "3-way batched Keccak (x3, AArch64 with SHA3)"
+      if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch == 'ARM64' }}
+      uses: ./.github/actions/functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        platform: native
+        cflags: "${{ inputs.extra_cflags }} -std=c11 -march=armv8.4-a+sha3 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
+        alloc: false # Requires custom config
+        sign_hook: false # Requires custom config
+        rng_fail: true
+    - name: "3-way batched Keccak (x3, C fallback)"
+      if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch != 'ARM64' }}
+      uses: ./.github/actions/functest
+      with:
+        gh_token: ${{ inputs.gh_token }}
+        platform: native
+        cflags: "${{ inputs.extra_cflags }} -std=c11 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+        func: true
+        kat: true
+        acvp: true
+        opt: ${{ inputs.opt }}
+        examples: false # Some examples use a custom config themselves
+        alloc: false # Requires custom config
+        sign_hook: false # Requires custom config
+        rng_fail: true
     - name: "Custom randombytes"
       if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'custom-randombytes') }}
       uses: ./.github/actions/functest

.github/workflows/bench.yml

@@ -167,7 +167,9 @@ jobs:
           - name: Graviton5
             ec2_instance_type: c9g.medium
             ec2_ami: ubuntu (aarch64)
-            archflags: -march=armv9-a+sha3 -DMLD_SYS_AARCH64_FAST_SHA3
+            # 3-way batched Keccak (x3). Not combined with MLD_SYS_AARCH64_FAST_SHA3,
+            # which selects the x2_v84a backend before MLD_CONFIG_USE_X3 is checked.
+            archflags: -march=armv9-a+sha3 -DMLD_CONFIG_USE_X3
             cflags: "-flto -DMLD_FORCE_AARCH64"
             ldflags: "-flto"
             perf: PERF

.github/workflows/ci.yml

@@ -187,7 +187,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar]
+        backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar, x3_v84a_scalar]
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -207,7 +207,8 @@ jobs:
           platform: native
           opt: 'opt'
           examples: 'false'
-          cflags: "-DMLDSA_DEBUG -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
+          # The x3 backend is only used by the samplers with MLD_CONFIG_USE_X3.
+          cflags: "-DMLDSA_DEBUG ${{ matrix.backend == 'x3_v84a_scalar' && '-DMLD_CONFIG_USE_X3 ' || '' }}-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
           ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
           check_namespace: 'false'
           extra_args: "--fips202-aarch64-backend ${{ matrix.backend }}"
@@ -549,7 +550,7 @@ jobs:
          - name: native-cap
            ids: native-cap-ON native-cap-OFF native-cap-ID_AA64PFR1_EL1 native-cap-CPUID_AVX2
          - name: asm-fips202
-           ids: no-asm serial-fips202
+           ids: no-asm serial-fips202 x3-keccak
          - name: nblocks
            ids: nblocks-1 nblocks-4 nblocks-6
          - name: reduced-api

.github/workflows/ci_ec2_reusable.yml

@@ -240,6 +240,8 @@ jobs:
             tests all --no-examples --opt opt --cflags="-DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_scalar.h\\\""
             make clean
             tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_v84a_scalar.h\\\""
+            make clean
+            tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_USE_X3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x3_v84a_scalar.h\\\""
   stop-ec2-runner:
     name: Stop instance (${{ inputs.ec2_instance_type }})
     permissions:

.github/workflows/hol_light.yml

@@ -110,6 +110,8 @@ jobs:
             needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"]
           - name: keccak_f1600_x2_v84a_aarch64_asm
             needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"]
+          - name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm
+            needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"]
           - name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm
             needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"]
           - name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm

.github/workflows/slothy.yml

@@ -61,6 +61,7 @@ jobs:
             dev/aarch64_clean/**
             dev/aarch64_opt/**
             dev/fips202/aarch64/**
+            dev/fips202/aarch64_symbolic/**
             mldsa/src/native/aarch64/**
             mldsa/src/fips202/native/aarch64/**
             flake.nix

BIBLIOGRAPHY.md

@@ -117,6 +117,7 @@ source code and documentation.
   - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h)
   - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h)
   - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h)
+  - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h)
 
 ### `FIPS202`
 
@@ -153,6 +154,7 @@ source code and documentation.
   - [mldsa/mldsa_native_config.h](mldsa/mldsa_native_config.h)
   - [mldsa/src/ct.h](mldsa/src/ct.h)
   - [mldsa/src/fips202/fips202.c](mldsa/src/fips202/fips202.c)
+  - [mldsa/src/fips202/fips202x3.c](mldsa/src/fips202/fips202x3.c)
   - [mldsa/src/fips202/fips202x4.c](mldsa/src/fips202/fips202x4.c)
   - [mldsa/src/packing.h](mldsa/src/packing.h)
   - [mldsa/src/poly.c](mldsa/src/poly.c)
@@ -183,6 +185,7 @@ source code and documentation.
   - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h)
   - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h)
   - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h)
+  - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h)
   - [test/src/test_sign_hook.c](test/src/test_sign_hook.c)
 
 ### `FIPS204_UPDATES`
@@ -224,6 +227,7 @@ source code and documentation.
   - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h)
   - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h)
   - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h)
+  - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h)
   - [test/src/test_sign_hook.c](test/src/test_sign_hook.c)
 
 ### `HYBRID`

dev/fips202/aarch64/auto.h

@@ -67,9 +67,17 @@
  */
 #if defined(__APPLE__) || defined(MLD_SYS_AARCH64_FAST_SHA3)
 #include "x2_v84a.h"
-#else
+#elif defined(MLD_CONFIG_USE_X3)
+/*
+ * 3-way hybrid (two states on Neon with SHA3, one on scalar registers, single
+ * pass), selected with MLD_CONFIG_USE_X3. Intended for cores that execute the
+ * SHA3 instructions on all SIMD pipes (e.g. Neoverse V3). When
+ * MLD_CONFIG_USE_X3 is not set, the x4 hybrid below is used.
+ */
+#include "x3_v84a_scalar.h"
+#else /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && MLD_CONFIG_USE_X3 */
 #include "x4_v8a_v84a_scalar.h"
-#endif
+#endif /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && !MLD_CONFIG_USE_X3 */
 
 #else /* __ARM_FEATURE_SHA3 */
 

dev/fips202/aarch64/src/Makefile

@@ -0,0 +1,156 @@
+# Copyright (c) The mlkem-native project authors
+# Copyright (c) The mldsa-native project authors
+# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+
+.PHONY: all purge
+.DEFAULT_GOAL := all
+
+keccak_f1600_x1_scalar_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S
+        # The last two configuration options are for fast testing (incl. CI).
+        # Remove when building a production version.
+
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $^ -o $@ \
+           -c reserved_regs="[x18,sp]" \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c constraints.allow_spills \
+	   -c constraints.minimize_spills \
+           -s mld_keccak_f1600_x1_scalar_loop \
+           -e mld_keccak_f1600_x1_scalar_end_loop \
+	   -c constraints.functional_only \
+	   -c objective_lower_bound=24
+
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $@ -o $@ \
+           -c reserved_regs="[x18,sp]" \
+	   -c variable_size \
+	   -c inputs_are_outputs \
+           -c outputs="[hint_STACK_LOC_COUNT]" \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c constraints.allow_spills \
+	   -c constraints.minimize_spills \
+           -s mld_keccak_f1600_x1_scalar_initial_start \
+           -e mld_keccak_f1600_x1_scalar_loop \
+	   -c constraints.functional_only \
+	   -c objective_lower_bound=24
+
+keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $^ -o $@                           \
+	   -c reserved_regs="[x18,sp]"        \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+	   -c absorb_spills=False \
+           -c outputs="[hint_STACK_OFFSET_COUNT]" \
+           -s mld_keccak_f1600_x4_v8a_scalar_hybrid_initial \
+           -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop
+
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $@ -o $@                           \
+	   -c reserved_regs="[x18,sp]"        \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+           -c outputs="[hint_STACK_OFFSET_COUNT]" \
+	   -c absorb_spills=False \
+           -s mld_keccak_f1600_x4_v8a_scalar_hybrid_loop \
+           -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop_end
+
+keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $^ -o $@                           \
+	   -c reserved_regs="[x18,sp]"        \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+	   -c absorb_spills=False \
+           -c outputs="[hint_STACK_OFFSET_COUNT]" \
+           -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial \
+           -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop
+
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $@ -o $@                           \
+	   -c reserved_regs="[x18,sp]"        \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+           -c outputs="[hint_STACK_OFFSET_COUNT]" \
+	   -c absorb_spills=False \
+           -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop \
+           -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end
+
+# x3 hybrid (2 Neon + 1 scalar, single pass): interleaving only; register
+# allocation and the instruction order within each stream are kept.
+keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $^ -o $@ \
+	   -c reserved_regs="[x18,sp]" \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c ignore_objective=True \
+	   -c constraints.functional_only=True \
+	   -c constraints.allow_reordering=False \
+	   -c constraints.allow_spills=True \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+	   -c absorb_spills=False \
+	   -c outputs="[hint_STACK_OFFSET_COUNT]" \
+	   -s mld_keccak_f1600_x3_v84a_scalar_hybrid_initial \
+	   -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop
+
+	slothy-cli Arm_AArch64 Arm_Cortex_A55 \
+	   $@ -o $@ \
+	   -c reserved_regs="[x18,sp]" \
+	   -c inputs_are_outputs \
+	   -c variable_size \
+	   -c constraints.stalls_first_attempt=64 \
+	   -c ignore_objective=True \
+	   -c constraints.functional_only=True \
+	   -c constraints.allow_reordering=False \
+	   -c constraints.allow_spills=True \
+	   -c split_heuristic \
+	   -c split_heuristic_repeat=0 \
+	   -c split_heuristic_preprocess_naive_interleaving \
+	   -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \
+	   -c split_heuristic_estimate_performance=False \
+	   -c absorb_spills=False \
+	   -c outputs="[hint_STACK_OFFSET_COUNT]" \
+	   -s mld_keccak_f1600_x3_v84a_scalar_hybrid_loop \
+	   -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end
+
+ALL=keccak_f1600_x1_scalar_aarch64_asm.S                \
+  keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S       \
+  keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S  \
+  keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S
+
+all: $(ALL)
+
+purge:
+	rm -rf $(ALL)

dev/fips202/aarch64/src/fips202_native_aarch64.h

@@ -66,4 +66,18 @@ __contract__(
   assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4))
 );
 
+/*
+ * 3-way hybrid Keccak-f1600: states 0 and 1 on Neon (Armv8.4-A SHA3), state 2
+ * on scalar registers with lazy rotations, interleaved 1:1 in a single pass.
+ */
+#define mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm \
+  MLD_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm)
+void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75],
+                                                        const uint64_t rc[24])
+__contract__(
+  requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3))
+  requires(rc == mld_keccakf1600_round_constants)
+  assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3))
+);
+
 #endif /* !MLD_DEV_FIPS202_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */