diff --git a/.github/actions/config-variations/action.yml b/.github/actions/config-variations/action.yml index f38fc08134..73359dcf09 100644 --- a/.github/actions/config-variations/action.yml +++ b/.github/actions/config-variations/action.yml @@ -8,7 +8,7 @@ inputs: description: 'GitHub token' required: true tests: - description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized' + description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, x3-keccak, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized' required: false default: 'all' opt: @@ -236,6 +236,40 @@ runs: alloc: false # Requires custom config sign_hook: false # Requires custom config rng_fail: true + # On AArch64, build with SHA3 so that the native x3 hybrid kernel is used; + # elsewhere, this tests the C fallback of the 3-way batched Keccak path. + - name: "3-way batched Keccak (x3, AArch64 with SHA3)" + if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch == 'ARM64' }} + uses: ./.github/actions/functest + with: + gh_token: ${{ inputs.gh_token }} + platform: native + cflags: "${{ inputs.extra_cflags }} -std=c11 -march=armv8.4-a+sha3 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + func: true + kat: true + acvp: true + opt: ${{ inputs.opt }} + examples: false # Some examples use a custom config themselves + alloc: false # Requires custom config + sign_hook: false # Requires custom config + rng_fail: true + - name: "3-way batched Keccak (x3, C fallback)" + if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch != 'ARM64' }} + uses: ./.github/actions/functest + with: + gh_token: ${{ inputs.gh_token }} + platform: native + cflags: "${{ inputs.extra_cflags }} -std=c11 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + func: true + kat: true + acvp: true + opt: ${{ inputs.opt }} + examples: false # Some examples use a custom config themselves + alloc: false # Requires custom config + sign_hook: false # Requires custom config + rng_fail: true - name: "Custom randombytes" if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'custom-randombytes') }} uses: ./.github/actions/functest diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 89c27b247c..0e609f3472 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -167,7 +167,9 @@ jobs: - name: Graviton5 ec2_instance_type: c9g.medium ec2_ami: ubuntu (aarch64) - archflags: -march=armv9-a+sha3 -DMLD_SYS_AARCH64_FAST_SHA3 + # 3-way batched Keccak (x3). Not combined with MLD_SYS_AARCH64_FAST_SHA3, + # which selects the x2_v84a backend before MLD_CONFIG_USE_X3 is checked. + archflags: -march=armv9-a+sha3 -DMLD_CONFIG_USE_X3 cflags: "-flto -DMLD_FORCE_AARCH64" ldflags: "-flto" perf: PERF diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbc9dc3347..50a5d03fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,7 +187,7 @@ jobs: strategy: fail-fast: false matrix: - backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar] + backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar, x3_v84a_scalar] runs-on: macos-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -207,7 +207,8 @@ jobs: platform: native opt: 'opt' examples: 'false' - cflags: "-DMLDSA_DEBUG -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + # The x3 backend is only used by the samplers with MLD_CONFIG_USE_X3. + cflags: "-DMLDSA_DEBUG ${{ matrix.backend == 'x3_v84a_scalar' && '-DMLD_CONFIG_USE_X3 ' || '' }}-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" check_namespace: 'false' extra_args: "--fips202-aarch64-backend ${{ matrix.backend }}" @@ -549,7 +550,7 @@ jobs: - name: native-cap ids: native-cap-ON native-cap-OFF native-cap-ID_AA64PFR1_EL1 native-cap-CPUID_AVX2 - name: asm-fips202 - ids: no-asm serial-fips202 + ids: no-asm serial-fips202 x3-keccak - name: nblocks ids: nblocks-1 nblocks-4 nblocks-6 - name: reduced-api diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml index ce247bcaaa..645ebe93ea 100644 --- a/.github/workflows/ci_ec2_reusable.yml +++ b/.github/workflows/ci_ec2_reusable.yml @@ -240,6 +240,8 @@ jobs: tests all --no-examples --opt opt --cflags="-DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_scalar.h\\\"" make clean tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_v84a_scalar.h\\\"" + make clean + tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_USE_X3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x3_v84a_scalar.h\\\"" stop-ec2-runner: name: Stop instance (${{ inputs.ec2_instance_type }}) permissions: diff --git a/.github/workflows/hol_light.yml b/.github/workflows/hol_light.yml index c364013574..eca809b169 100644 --- a/.github/workflows/hol_light.yml +++ b/.github/workflows/hol_light.yml @@ -110,6 +110,8 @@ jobs: needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x2_v84a_aarch64_asm needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"] + - name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm diff --git a/.github/workflows/slothy.yml b/.github/workflows/slothy.yml index 85adf720b7..ba29d88f58 100644 --- a/.github/workflows/slothy.yml +++ b/.github/workflows/slothy.yml @@ -61,6 +61,7 @@ jobs: dev/aarch64_clean/** dev/aarch64_opt/** dev/fips202/aarch64/** + dev/fips202/aarch64_symbolic/** mldsa/src/native/aarch64/** mldsa/src/fips202/native/aarch64/** flake.nix diff --git a/BIBLIOGRAPHY.md b/BIBLIOGRAPHY.md index f397dff750..22724f97d9 100644 --- a/BIBLIOGRAPHY.md +++ b/BIBLIOGRAPHY.md @@ -117,6 +117,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) ### `FIPS202` @@ -153,6 +154,7 @@ source code and documentation. - [mldsa/mldsa_native_config.h](mldsa/mldsa_native_config.h) - [mldsa/src/ct.h](mldsa/src/ct.h) - [mldsa/src/fips202/fips202.c](mldsa/src/fips202/fips202.c) + - [mldsa/src/fips202/fips202x3.c](mldsa/src/fips202/fips202x3.c) - [mldsa/src/fips202/fips202x4.c](mldsa/src/fips202/fips202x4.c) - [mldsa/src/packing.h](mldsa/src/packing.h) - [mldsa/src/poly.c](mldsa/src/poly.c) @@ -183,6 +185,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) - [test/src/test_sign_hook.c](test/src/test_sign_hook.c) ### `FIPS204_UPDATES` @@ -224,6 +227,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) - [test/src/test_sign_hook.c](test/src/test_sign_hook.c) ### `HYBRID` diff --git a/dev/fips202/aarch64/auto.h b/dev/fips202/aarch64/auto.h index 98045b2dbe..3cfdcf2473 100644 --- a/dev/fips202/aarch64/auto.h +++ b/dev/fips202/aarch64/auto.h @@ -67,9 +67,17 @@ */ #if defined(__APPLE__) || defined(MLD_SYS_AARCH64_FAST_SHA3) #include "x2_v84a.h" -#else +#elif defined(MLD_CONFIG_USE_X3) +/* + * 3-way hybrid (two states on Neon with SHA3, one on scalar registers, single + * pass), selected with MLD_CONFIG_USE_X3. Intended for cores that execute the + * SHA3 instructions on all SIMD pipes (e.g. Neoverse V3). When + * MLD_CONFIG_USE_X3 is not set, the x4 hybrid below is used. + */ +#include "x3_v84a_scalar.h" +#else /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && MLD_CONFIG_USE_X3 */ #include "x4_v8a_v84a_scalar.h" -#endif +#endif /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && !MLD_CONFIG_USE_X3 */ #else /* __ARM_FEATURE_SHA3 */ diff --git a/dev/fips202/aarch64/src/Makefile b/dev/fips202/aarch64/src/Makefile new file mode 100644 index 0000000000..ea95b7c5db --- /dev/null +++ b/dev/fips202/aarch64/src/Makefile @@ -0,0 +1,156 @@ +# Copyright (c) The mlkem-native project authors +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +.PHONY: all purge +.DEFAULT_GOAL := all + +keccak_f1600_x1_scalar_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S + # The last two configuration options are for fast testing (incl. CI). + # Remove when building a production version. + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c constraints.allow_spills \ + -c constraints.minimize_spills \ + -s mld_keccak_f1600_x1_scalar_loop \ + -e mld_keccak_f1600_x1_scalar_end_loop \ + -c constraints.functional_only \ + -c objective_lower_bound=24 + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c variable_size \ + -c inputs_are_outputs \ + -c outputs="[hint_STACK_LOC_COUNT]" \ + -c constraints.stalls_first_attempt=64 \ + -c constraints.allow_spills \ + -c constraints.minimize_spills \ + -s mld_keccak_f1600_x1_scalar_initial_start \ + -e mld_keccak_f1600_x1_scalar_loop \ + -c constraints.functional_only \ + -c objective_lower_bound=24 + +keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x4_v8a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -c absorb_spills=False \ + -s mld_keccak_f1600_x4_v8a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop_end + +keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -c absorb_spills=False \ + -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end + +# x3 hybrid (2 Neon + 1 scalar, single pass): interleaving only; register +# allocation and the instruction order within each stream are kept. +keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c ignore_objective=True \ + -c constraints.functional_only=True \ + -c constraints.allow_reordering=False \ + -c constraints.allow_spills=True \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x3_v84a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c ignore_objective=True \ + -c constraints.functional_only=True \ + -c constraints.allow_reordering=False \ + -c constraints.allow_spills=True \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x3_v84a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end + +ALL=keccak_f1600_x1_scalar_aarch64_asm.S \ + keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S \ + keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S \ + keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S + +all: $(ALL) + +purge: + rm -rf $(ALL) diff --git a/dev/fips202/aarch64/src/fips202_native_aarch64.h b/dev/fips202/aarch64/src/fips202_native_aarch64.h index 3939c2359c..410d70e10a 100644 --- a/dev/fips202/aarch64/src/fips202_native_aarch64.h +++ b/dev/fips202/aarch64/src/fips202_native_aarch64.h @@ -66,4 +66,18 @@ __contract__( assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) ); +/* + * 3-way hybrid Keccak-f1600: states 0 and 1 on Neon (Armv8.4-A SHA3), state 2 + * on scalar registers with lazy rotations, interleaved 1:1 in a single pass. + */ +#define mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm \ + MLD_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) +); + #endif /* !MLD_DEV_FIPS202_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..3724e589c1 --- /dev/null +++ b/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,1598 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/* ***************** REGISTER ALLOCATIONS ****************** */ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + cur_const .req x26 + + /* Mapping of Keccak-f1600 SIMD state to vector registers + * at the beginning and end of each round. Identical to x4: + * two independent states are packed into the 2 lanes of each + * 128-bit V register. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Mapping of Keccak-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/* *********************** MACROS *************************** */ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +/* Vector states are state0 (at input_addr) and state1 (at input_addr+0xc8). + * Identical layout to x4. */ +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +/* The single scalar state is state2, at offset (2)*25*8 from input_addr. + * x3 only ever uses idx=0 (there is no second scalar pass). */ +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// Scalar round macros are IDENTICAL to the x4 kernel; they operate on the +// single scalar state held in GPRs. +.macro scalar_round_initial + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #1 + bic x9, x0, x16, ror #9 + str x24, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 +.endm + +// Neon round: uses the single-instruction Armv8.4-A SHA3 forms (eor3, rax1, +// xar, bcax; the "_m0" macros) throughout. Requires __ARM_FEATURE_SHA3 +// (guarded above). +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m0 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m0 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m0 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m0 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m0 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m0 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m0 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m0 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m0 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m0 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m0 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m0 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m0 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m0 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m0 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m0 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m0 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m0 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m0 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m0 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m0 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m0 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m0 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m0 vAko, vAko_, vAka_, vAku_ + bcax_m0 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m0 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m0 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m0 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m0 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m0 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m0 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m0 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #62 + eor x30, x30, x26, ror #57 + ror x26, x26, #58 + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #56 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, LSL #3] + ldr x9, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + add x25, x25, #1 + str x25, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input (state0, state1) + load_input_scalar 0 // Scalar input (state2) + + // Single pass, 1:1 interleave. + // Initial iteration: scalar_round_initial does round 0 of the scalar state + // and sets the round counter to 1; vector_round does round 0 of the two + // vector states. Then the loop runs rounds 1..23 (23 iterations), one + // scalar + one vector round each, for a total of 24 rounds on all three + // states. + mld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + // Instructions: 176 + // Expected cycles: 88 + // Expected IPC: 2.00 + + // ---------------------------------- cycle (expected) -----------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|------------ + eor x30, x24, x25 // *....................................................................................... // @slothy:interleaving_class=0 + eor x27, x9, x10 // *....................................................................................... // @slothy:interleaving_class=0 + eor3 v30.16B, v0.16B, v5.16B, v10.16B // .*...................................................................................... // @slothy:interleaving_class=1 + eor3 v30.16B, v30.16B, v15.16B, v20.16B // .*...................................................................................... // @slothy:interleaving_class=1 + eor x0, x30, x21 // ..*..................................................................................... // @slothy:interleaving_class=0 + eor x26, x27, x6 // ..*..................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v1.16B, v6.16B, v11.16B // ...*.................................................................................... // @slothy:interleaving_class=1 + eor x27, x26, x7 // ...*.................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v29.16B, v16.16B, v21.16B // ....*................................................................................... // @slothy:interleaving_class=1 + eor x29, x0, x22 // ....*................................................................................... // @slothy:interleaving_class=0 + eor x26, x29, x23 // .....*.................................................................................. // @slothy:interleaving_class=0 + eor3 v28.16B, v2.16B, v7.16B, v12.16B // .....*.................................................................................. // @slothy:interleaving_class=1 + eor x29, x4, x5 // ......*................................................................................. // @slothy:interleaving_class=0 + eor3 v28.16B, v28.16B, v17.16B, v22.16B // ......*................................................................................. // @slothy:interleaving_class=1 + eor x30, x29, x1 // .......*................................................................................ // @slothy:interleaving_class=0 + eor x0, x27, x8 // .......*................................................................................ // @slothy:interleaving_class=0 + eor3 v27.16B, v3.16B, v8.16B, v13.16B // ........*............................................................................... // @slothy:interleaving_class=1 + eor x29, x30, x2 // ........*............................................................................... // @slothy:interleaving_class=0 + eor3 v27.16B, v27.16B, v18.16B, v23.16B // .........*.............................................................................. // @slothy:interleaving_class=1 + eor x30, x19, x20 // .........*.............................................................................. // @slothy:interleaving_class=0 + eor x30, x30, x16 // ..........*............................................................................. // @slothy:interleaving_class=0 + eor3 v26.16B, v4.16B, v9.16B, v14.16B // ..........*............................................................................. // @slothy:interleaving_class=1 + eor x27, x26, x0, ror #63 // ...........*............................................................................ // @slothy:interleaving_class=0 + eor3 v26.16B, v26.16B, v19.16B, v24.16B // ...........*............................................................................ // @slothy:interleaving_class=1 + eor x4, x4, x27 // ............*........................................................................... // @slothy:interleaving_class=0 + eor x30, x30, x17 // ............*........................................................................... // @slothy:interleaving_class=0 + rax1 v25.2D, v30.2D, v28.2D // .............*.......................................................................... // @slothy:interleaving_class=1 + eor x30, x30, x28 // .............*.......................................................................... // @slothy:interleaving_class=0 + eor x29, x29, x3 // ..............*......................................................................... // @slothy:interleaving_class=0 + rax1 v28.2D, v28.2D, v26.2D // ..............*......................................................................... // @slothy:interleaving_class=1 + eor x0, x0, x30, ror #63 // ...............*........................................................................ // @slothy:interleaving_class=0 + rax1 v26.2D, v26.2D, v29.2D // ...............*........................................................................ // @slothy:interleaving_class=1 + eor x30, x30, x29, ror #63 // ................*....................................................................... // @slothy:interleaving_class=0 + eor x22, x22, x30 // ................*....................................................................... // @slothy:interleaving_class=0 + rax1 v29.2D, v29.2D, v27.2D // .................*...................................................................... // @slothy:interleaving_class=1 + eor x23, x23, x30 // .................*...................................................................... // @slothy:interleaving_class=0 + rax1 v27.2D, v27.2D, v30.2D // ..................*..................................................................... // @slothy:interleaving_class=1 + str x23, [sp, #STACK_LOC_0] // ..................*..................................................................... // @slothy:writes=stack_0 // @slothy:interleaving_class=0 + eor x23, x14, x15 // ...................*.................................................................... // @slothy:interleaving_class=0 + eor v30.16B, v0.16B, v26.16B // ...................*.................................................................... // @slothy:interleaving_class=1 + eor x14, x14, x0 // ....................*................................................................... // @slothy:interleaving_class=0 + xar v0.2D, v2.2D, v29.2D, #2 // ....................*................................................................... // @slothy:interleaving_class=1 + eor x23, x23, x11 // .....................*.................................................................. // @slothy:interleaving_class=0 + eor x15, x15, x0 // .....................*.................................................................. // @slothy:interleaving_class=0 + xar v2.2D, v12.2D, v29.2D, #21 // ......................*................................................................. // @slothy:interleaving_class=1 + eor x1, x1, x27 // ......................*................................................................. // @slothy:interleaving_class=0 + xar v12.2D, v13.2D, v28.2D, #39 // .......................*................................................................ // @slothy:interleaving_class=1 + eor x23, x23, x12 // .......................*................................................................ // @slothy:interleaving_class=0 + eor x23, x23, x13 // ........................*............................................................... // @slothy:interleaving_class=0 + xar v13.2D, v19.2D, v27.2D, #56 // ........................*............................................................... // @slothy:interleaving_class=1 + eor x11, x11, x0 // .........................*.............................................................. // @slothy:interleaving_class=0 + eor x29, x29, x23, ror #63 // .........................*.............................................................. // @slothy:interleaving_class=0 + xar v19.2D, v23.2D, v28.2D, #8 // ..........................*............................................................. // @slothy:interleaving_class=1 + eor x23, x23, x26, ror #63 // ..........................*............................................................. // @slothy:interleaving_class=0 + xar v23.2D, v15.2D, v26.2D, #23 // ...........................*............................................................ // @slothy:interleaving_class=1 + eor x26, x13, x0 // ...........................*............................................................ // @slothy:interleaving_class=0 + eor x13, x28, x23 // ............................*........................................................... // @slothy:interleaving_class=0 + xar v15.2D, v1.2D, v25.2D, #63 // ............................*........................................................... // @slothy:interleaving_class=1 + eor x28, x24, x30 // .............................*.......................................................... // @slothy:interleaving_class=0 + xar v1.2D, v8.2D, v28.2D, #9 // .............................*.......................................................... // @slothy:interleaving_class=1 + eor x24, x16, x23 // ..............................*......................................................... // @slothy:interleaving_class=0 + eor x16, x21, x30 // ..............................*......................................................... // @slothy:interleaving_class=0 + xar v8.2D, v16.2D, v25.2D, #19 // ...............................*........................................................ // @slothy:interleaving_class=1 + eor x21, x25, x30 // ...............................*........................................................ // @slothy:interleaving_class=0 + xar v16.2D, v7.2D, v29.2D, #58 // ................................*....................................................... // @slothy:interleaving_class=1 + eor x30, x19, x23 // ................................*....................................................... // @slothy:interleaving_class=0 + eor x19, x20, x23 // .................................*...................................................... // @slothy:interleaving_class=0 + xar v7.2D, v10.2D, v26.2D, #61 // .................................*...................................................... // @slothy:interleaving_class=1 + eor x20, x17, x23 // ..................................*..................................................... // @slothy:interleaving_class=0 + xar v10.2D, v3.2D, v28.2D, #36 // ..................................*..................................................... // @slothy:interleaving_class=1 + eor x17, x12, x0 // ...................................*.................................................... // @slothy:interleaving_class=0 + eor x0, x2, x27 // ...................................*.................................................... // @slothy:interleaving_class=0 + xar v3.2D, v18.2D, v28.2D, #43 // ....................................*................................................... // @slothy:interleaving_class=1 + eor x2, x6, x29 // ....................................*................................................... // @slothy:interleaving_class=0 + xar v18.2D, v17.2D, v29.2D, #49 // .....................................*.................................................. // @slothy:interleaving_class=1 + eor x6, x8, x29 // .....................................*.................................................. // @slothy:interleaving_class=0 + bic x8, x28, x13, ror #47 // ......................................*................................................. // @slothy:interleaving_class=0 + xar v17.2D, v11.2D, v25.2D, #54 // ......................................*................................................. // @slothy:interleaving_class=1 + eor x12, x3, x27 // .......................................*................................................ // @slothy:interleaving_class=0 + bic x3, x13, x17, ror #19 // .......................................*................................................ // @slothy:interleaving_class=0 + xar v11.2D, v9.2D, v27.2D, #44 // ........................................*............................................... // @slothy:interleaving_class=1 + eor x5, x5, x27 // ........................................*............................................... // @slothy:interleaving_class=0 + xar v9.2D, v22.2D, v29.2D, #3 // .........................................*.............................................. // @slothy:interleaving_class=1 + ldr x27, [sp, #STACK_LOC_0] // .........................................*.............................................. // @slothy:reads=stack_0 // @slothy:interleaving_class=0 + bic x25, x17, x2, ror #5 // ..........................................*............................................. // @slothy:interleaving_class=0 + xar v22.2D, v14.2D, v27.2D, #25 // ..........................................*............................................. // @slothy:interleaving_class=1 + eor x9, x9, x29 // ...........................................*............................................ // @slothy:interleaving_class=0 + xar v14.2D, v20.2D, v26.2D, #46 // ...........................................*............................................ // @slothy:interleaving_class=1 + eor x23, x25, x5, ror #52 // ............................................*........................................... // @slothy:interleaving_class=0 + eor x3, x3, x2, ror #24 // ............................................*........................................... // @slothy:interleaving_class=0 + xar v20.2D, v4.2D, v27.2D, #37 // .............................................*.......................................... // @slothy:interleaving_class=1 + eor x8, x8, x17, ror #2 // .............................................*.......................................... // @slothy:interleaving_class=0 + xar v4.2D, v24.2D, v27.2D, #50 // ..............................................*......................................... // @slothy:interleaving_class=1 + eor x17, x10, x29 // ..............................................*......................................... // @slothy:interleaving_class=0 + bic x25, x12, x22, ror #47 // ...............................................*........................................ // @slothy:interleaving_class=0 + xar v24.2D, v21.2D, v25.2D, #62 // ...............................................*........................................ // @slothy:interleaving_class=1 + eor x29, x7, x29 // ................................................*....................................... // @slothy:interleaving_class=0 + xar v21.2D, v5.2D, v26.2D, #28 // ................................................*....................................... // @slothy:interleaving_class=1 + bic x10, x4, x27, ror #2 // .................................................*...................................... // @slothy:interleaving_class=0 + bic x7, x5, x28, ror #10 // .................................................*...................................... // @slothy:interleaving_class=0 + xar v27.2D, v6.2D, v25.2D, #20 // ..................................................*..................................... // @slothy:interleaving_class=1 + eor x10, x10, x20, ror #50 // ..................................................*..................................... // @slothy:interleaving_class=0 + eor x13, x7, x13, ror #57 // ...................................................*.................................... // @slothy:interleaving_class=0 + bcax v5.16B, v10.16B, v7.16B, v11.16B // ...................................................*.................................... // @slothy:interleaving_class=1 + bic x7, x2, x5, ror #47 // ....................................................*................................... // @slothy:interleaving_class=0 + bcax v6.16B, v11.16B, v8.16B, v7.16B // ....................................................*................................... // @slothy:interleaving_class=1 + eor x2, x25, x24, ror #39 // .....................................................*.................................. // @slothy:interleaving_class=0 + bic x25, x20, x11, ror #57 // .....................................................*.................................. // @slothy:interleaving_class=0 + bcax v7.16B, v7.16B, v9.16B, v8.16B // ......................................................*................................. // @slothy:interleaving_class=1 + bic x5, x17, x4, ror #25 // ......................................................*................................. // @slothy:interleaving_class=0 + bcax v8.16B, v8.16B, v10.16B, v9.16B // .......................................................*................................ // @slothy:interleaving_class=1 + eor x25, x25, x17, ror #53 // .......................................................*................................ // @slothy:interleaving_class=0 + bic x17, x11, x17, ror #60 // ........................................................*............................... // @slothy:interleaving_class=0 + bcax v9.16B, v9.16B, v11.16B, v10.16B // ........................................................*............................... // @slothy:interleaving_class=1 + eor x28, x7, x28, ror #57 // .........................................................*.............................. // @slothy:interleaving_class=0 + bcax v10.16B, v15.16B, v12.16B, v16.16B // .........................................................*.............................. // @slothy:interleaving_class=1 + bic x7, x9, x12, ror #42 // ..........................................................*............................. // @slothy:interleaving_class=0 + eor x7, x7, x22, ror #25 // ..........................................................*............................. // @slothy:interleaving_class=0 + bcax v11.16B, v16.16B, v13.16B, v12.16B // ...........................................................*............................ // @slothy:interleaving_class=1 + bic x22, x22, x24, ror #56 // ...........................................................*............................ // @slothy:interleaving_class=0 + bcax v12.16B, v12.16B, v14.16B, v13.16B // ............................................................*........................... // @slothy:interleaving_class=1 + bic x24, x24, x15, ror #31 // ............................................................*........................... // @slothy:interleaving_class=0 + eor x22, x22, x15, ror #23 // .............................................................*.......................... // @slothy:interleaving_class=0 + bcax v13.16B, v13.16B, v15.16B, v14.16B // .............................................................*.......................... // @slothy:interleaving_class=1 + bic x20, x27, x20, ror #48 // ..............................................................*......................... // @slothy:interleaving_class=0 + bcax v14.16B, v14.16B, v16.16B, v15.16B // ..............................................................*......................... // @slothy:interleaving_class=1 + bic x15, x15, x9, ror #16 // ...............................................................*........................ // @slothy:interleaving_class=0 + eor x12, x15, x12, ror #58 // ...............................................................*........................ // @slothy:interleaving_class=0 + bcax v15.16B, v20.16B, v17.16B, v21.16B // ................................................................*....................... // @slothy:interleaving_class=1 + eor x15, x5, x27, ror #27 // ................................................................*....................... // @slothy:interleaving_class=0 + eor x5, x20, x11, ror #41 // .................................................................*...................... // @slothy:interleaving_class=0 + bcax v16.16B, v21.16B, v18.16B, v17.16B // .................................................................*...................... // @slothy:interleaving_class=1 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ..................................................................*..................... // @slothy:interleaving_class=0 + bcax v17.16B, v17.16B, v19.16B, v18.16B // ..................................................................*..................... // @slothy:interleaving_class=1 + eor x20, x17, x4, ror #21 // ...................................................................*.................... // @slothy:interleaving_class=0 + eor x17, x24, x9, ror #47 // ...................................................................*.................... // @slothy:interleaving_class=0 + bcax v18.16B, v18.16B, v20.16B, v19.16B // ....................................................................*................... // @slothy:interleaving_class=1 + mov x24, #1 // ....................................................................*................... // @slothy:interleaving_class=0 + bcax v19.16B, v19.16B, v21.16B, v20.16B // .....................................................................*.................. // @slothy:interleaving_class=1 + bic x9, x0, x16, ror #9 // .....................................................................*.................. // @slothy:interleaving_class=0 + str x24, [sp, #STACK_OFFSET_COUNT] // ......................................................................*................. // @slothy:writes=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v20.16B, v0.16B, v22.16B, v1.16B // ......................................................................*................. // @slothy:interleaving_class=1 + bic x24, x29, x1, ror #44 // .......................................................................*................ // @slothy:interleaving_class=0 + bcax v21.16B, v1.16B, v23.16B, v22.16B // .......................................................................*................ // @slothy:interleaving_class=1 + bic x27, x1, x21, ror #50 // ........................................................................*............... // @slothy:interleaving_class=0 + bic x4, x26, x29, ror #63 // ........................................................................*............... // @slothy:interleaving_class=0 + bcax v22.16B, v22.16B, v24.16B, v23.16B // .........................................................................*.............. // @slothy:interleaving_class=1 + eor x1, x1, x4, ror #21 // .........................................................................*.............. // @slothy:interleaving_class=0 + bcax v23.16B, v23.16B, v0.16B, v24.16B // ..........................................................................*............. // @slothy:interleaving_class=1 + ldr x11, [x11] // ..........................................................................*............. // @slothy:interleaving_class=0 + bic x4, x21, x30, ror #57 // ...........................................................................*............ // @slothy:interleaving_class=0 + bcax v24.16B, v24.16B, v1.16B, v0.16B // ...........................................................................*............ // @slothy:interleaving_class=1 + eor x21, x24, x21, ror #30 // ............................................................................*........... // @slothy:interleaving_class=0 + eor x24, x9, x19, ror #44 // ............................................................................*........... // @slothy:interleaving_class=0 + bcax v0.16B, v30.16B, v2.16B, v27.16B // .............................................................................*.......... // @slothy:interleaving_class=1 + bic x9, x14, x6, ror #5 // .............................................................................*.......... // @slothy:interleaving_class=0 + bcax v1.16B, v27.16B, v3.16B, v2.16B // ..............................................................................*......... // @slothy:interleaving_class=1 + eor x9, x9, x0, ror #43 // ..............................................................................*......... // @slothy:interleaving_class=0 + bic x0, x6, x0, ror #38 // ...............................................................................*........ // @slothy:interleaving_class=0 + bcax v2.16B, v2.16B, v4.16B, v3.16B // ...............................................................................*........ // @slothy:interleaving_class=1 + eor x1, x1, x11 // ................................................................................*....... // @slothy:interleaving_class=0 + bcax v3.16B, v3.16B, v30.16B, v4.16B // ................................................................................*....... // @slothy:interleaving_class=1 + eor x11, x4, x26, ror #35 // .................................................................................*...... // @slothy:interleaving_class=0 + eor x4, x0, x16, ror #47 // .................................................................................*...... // @slothy:interleaving_class=0 + bcax v4.16B, v4.16B, v27.16B, v30.16B // ..................................................................................*..... // @slothy:interleaving_class=1 + bic x0, x16, x19, ror #35 // ..................................................................................*..... // @slothy:interleaving_class=0 + eor x16, x27, x30, ror #43 // ...................................................................................*.... // @slothy:interleaving_class=0 + bic x27, x30, x26, ror #42 // ...................................................................................*.... // @slothy:interleaving_class=0 + ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ....................................................................................*... // @slothy:reads=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + ld1r {v28.2D}, [x30], #8 // ....................................................................................*... // @slothy:interleaving_class=1 + bic x26, x19, x14, ror #41 // .....................................................................................*.. // @slothy:interleaving_class=0 + str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .....................................................................................*.. // @slothy:writes=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + eor x19, x0, x14, ror #12 // ......................................................................................*. // @slothy:interleaving_class=0 + eor x14, x26, x6, ror #46 // ......................................................................................*. // @slothy:interleaving_class=0 + eor v0.16B, v0.16B, v28.16B // .......................................................................................* // @slothy:interleaving_class=1 + eor x6, x27, x29, ror #41 // .......................................................................................* // @slothy:interleaving_class=0 + + // ---------------------------------- cycle (expected) -----------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|------------ + // eor x30, x24, x25 // *....................................................................................... + // eor x27, x9, x10 // *....................................................................................... + // eor x0, x30, x21 // ..*..................................................................................... + // eor x26, x27, x6 // ..*..................................................................................... + // eor x27, x26, x7 // ...*.................................................................................... + // eor x29, x0, x22 // ....*................................................................................... + // eor x26, x29, x23 // .....*.................................................................................. + // eor x29, x4, x5 // ......*................................................................................. + // eor x30, x29, x1 // .......*................................................................................ + // eor x0, x27, x8 // .......*................................................................................ + // eor x29, x30, x2 // ........*............................................................................... + // eor x30, x19, x20 // .........*.............................................................................. + // eor x30, x30, x16 // ..........*............................................................................. + // eor x27, x26, x0, ror #63 // ...........*............................................................................ + // eor x4, x4, x27 // ............*........................................................................... + // eor x30, x30, x17 // ............*........................................................................... + // eor x30, x30, x28 // .............*.......................................................................... + // eor x29, x29, x3 // ..............*......................................................................... + // eor x0, x0, x30, ror #63 // ...............*........................................................................ + // eor x30, x30, x29, ror #63 // ................*....................................................................... + // eor x22, x22, x30 // ................*....................................................................... + // eor x23, x23, x30 // .................*...................................................................... + // str x23, [sp, #STACK_LOC_0] // ..................*..................................................................... + // eor x23, x14, x15 // ...................*.................................................................... + // eor x14, x14, x0 // ....................*................................................................... + // eor x23, x23, x11 // .....................*.................................................................. + // eor x15, x15, x0 // .....................*.................................................................. + // eor x1, x1, x27 // ......................*................................................................. + // eor x23, x23, x12 // .......................*................................................................ + // eor x23, x23, x13 // ........................*............................................................... + // eor x11, x11, x0 // .........................*.............................................................. + // eor x29, x29, x23, ror #63 // .........................*.............................................................. + // eor x23, x23, x26, ror #63 // ..........................*............................................................. + // eor x26, x13, x0 // ...........................*............................................................ + // eor x13, x28, x23 // ............................*........................................................... + // eor x28, x24, x30 // .............................*.......................................................... + // eor x24, x16, x23 // ..............................*......................................................... + // eor x16, x21, x30 // ..............................*......................................................... + // eor x21, x25, x30 // ...............................*........................................................ + // eor x30, x19, x23 // ................................*....................................................... + // eor x19, x20, x23 // .................................*...................................................... + // eor x20, x17, x23 // ..................................*..................................................... + // eor x17, x12, x0 // ...................................*.................................................... + // eor x0, x2, x27 // ...................................*.................................................... + // eor x2, x6, x29 // ....................................*................................................... + // eor x6, x8, x29 // .....................................*.................................................. + // bic x8, x28, x13, ror #47 // ......................................*................................................. + // eor x12, x3, x27 // .......................................*................................................ + // bic x3, x13, x17, ror #19 // .......................................*................................................ + // eor x5, x5, x27 // ........................................*............................................... + // ldr x27, [sp, #STACK_LOC_0] // .........................................*.............................................. + // bic x25, x17, x2, ror #5 // ..........................................*............................................. + // eor x9, x9, x29 // ...........................................*............................................ + // eor x23, x25, x5, ror #52 // ............................................*........................................... + // eor x3, x3, x2, ror #24 // ............................................*........................................... + // eor x8, x8, x17, ror #2 // .............................................*.......................................... + // eor x17, x10, x29 // ..............................................*......................................... + // bic x25, x12, x22, ror #47 // ...............................................*........................................ + // eor x29, x7, x29 // ................................................*....................................... + // bic x10, x4, x27, ror #2 // .................................................*...................................... + // bic x7, x5, x28, ror #10 // .................................................*...................................... + // eor x10, x10, x20, ror #50 // ..................................................*..................................... + // eor x13, x7, x13, ror #57 // ...................................................*.................................... + // bic x7, x2, x5, ror #47 // ....................................................*................................... + // eor x2, x25, x24, ror #39 // .....................................................*.................................. + // bic x25, x20, x11, ror #57 // .....................................................*.................................. + // bic x5, x17, x4, ror #25 // ......................................................*................................. + // eor x25, x25, x17, ror #53 // .......................................................*................................ + // bic x17, x11, x17, ror #60 // ........................................................*............................... + // eor x28, x7, x28, ror #57 // .........................................................*.............................. + // bic x7, x9, x12, ror #42 // ..........................................................*............................. + // eor x7, x7, x22, ror #25 // ..........................................................*............................. + // bic x22, x22, x24, ror #56 // ...........................................................*............................ + // bic x24, x24, x15, ror #31 // ............................................................*........................... + // eor x22, x22, x15, ror #23 // .............................................................*.......................... + // bic x20, x27, x20, ror #48 // ..............................................................*......................... + // bic x15, x15, x9, ror #16 // ...............................................................*........................ + // eor x12, x15, x12, ror #58 // ...............................................................*........................ + // eor x15, x5, x27, ror #27 // ................................................................*....................... + // eor x5, x20, x11, ror #41 // .................................................................*...................... + // ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ..................................................................*..................... + // eor x20, x17, x4, ror #21 // ...................................................................*.................... + // eor x17, x24, x9, ror #47 // ...................................................................*.................... + // mov x24, #1 // ....................................................................*................... + // bic x9, x0, x16, ror #9 // .....................................................................*.................. + // str x24, [sp, #STACK_OFFSET_COUNT] // ......................................................................*................. + // bic x24, x29, x1, ror #44 // .......................................................................*................ + // bic x27, x1, x21, ror #50 // ........................................................................*............... + // bic x4, x26, x29, ror #63 // ........................................................................*............... + // eor x1, x1, x4, ror #21 // .........................................................................*.............. + // ldr x11, [x11] // ..........................................................................*............. + // bic x4, x21, x30, ror #57 // ...........................................................................*............ + // eor x21, x24, x21, ror #30 // ............................................................................*........... + // eor x24, x9, x19, ror #44 // ............................................................................*........... + // bic x9, x14, x6, ror #5 // .............................................................................*.......... + // eor x9, x9, x0, ror #43 // ..............................................................................*......... + // bic x0, x6, x0, ror #38 // ...............................................................................*........ + // eor x1, x1, x11 // ................................................................................*....... + // eor x11, x4, x26, ror #35 // .................................................................................*...... + // eor x4, x0, x16, ror #47 // .................................................................................*...... + // bic x0, x16, x19, ror #35 // ..................................................................................*..... + // eor x16, x27, x30, ror #43 // ...................................................................................*.... + // bic x27, x30, x26, ror #42 // ...................................................................................*.... + // bic x26, x19, x14, ror #41 // .....................................................................................*.. + // eor x19, x0, x14, ror #12 // ......................................................................................*. + // eor x14, x26, x6, ror #46 // ......................................................................................*. + // eor x6, x27, x29, ror #41 // .......................................................................................* + // eor3 v30.16b, v0.16b, v5.16b, v10.16b // .*...................................................................................... + // eor3 v30.16b, v30.16b, v15.16b, v20.16b // .*...................................................................................... + // eor3 v29.16b, v1.16b, v6.16b, v11.16b // ...*.................................................................................... + // eor3 v29.16b, v29.16b, v16.16b, v21.16b // ....*................................................................................... + // eor3 v28.16b, v2.16b, v7.16b, v12.16b // .....*.................................................................................. + // eor3 v28.16b, v28.16b, v17.16b, v22.16b // ......*................................................................................. + // eor3 v27.16b, v3.16b, v8.16b, v13.16b // ........*............................................................................... + // eor3 v27.16b, v27.16b, v18.16b, v23.16b // .........*.............................................................................. + // eor3 v26.16b, v4.16b, v9.16b, v14.16b // ..........*............................................................................. + // eor3 v26.16b, v26.16b, v19.16b, v24.16b // ...........*............................................................................ + // rax1 v25.2d, v30.2d, v28.2d // .............*.......................................................................... + // rax1 v28.2d, v28.2d, v26.2d // ..............*......................................................................... + // rax1 v26.2d, v26.2d, v29.2d // ...............*........................................................................ + // rax1 v29.2d, v29.2d, v27.2d // .................*...................................................................... + // rax1 v27.2d, v27.2d, v30.2d // ..................*..................................................................... + // eor v30.16b, v0.16b, v26.16b // ...................*.................................................................... + // xar v0.2d, v2.2d, v29.2d, #2 // ....................*................................................................... + // xar v2.2d, v12.2d, v29.2d, #21 // ......................*................................................................. + // xar v12.2d, v13.2d, v28.2d, #39 // .......................*................................................................ + // xar v13.2d, v19.2d, v27.2d, #56 // ........................*............................................................... + // xar v19.2d, v23.2d, v28.2d, #8 // ..........................*............................................................. + // xar v23.2d, v15.2d, v26.2d, #23 // ...........................*............................................................ + // xar v15.2d, v1.2d, v25.2d, #63 // ............................*........................................................... + // xar v1.2d, v8.2d, v28.2d, #9 // .............................*.......................................................... + // xar v8.2d, v16.2d, v25.2d, #19 // ...............................*........................................................ + // xar v16.2d, v7.2d, v29.2d, #58 // ................................*....................................................... + // xar v7.2d, v10.2d, v26.2d, #61 // .................................*...................................................... + // xar v10.2d, v3.2d, v28.2d, #36 // ..................................*..................................................... + // xar v3.2d, v18.2d, v28.2d, #43 // ....................................*................................................... + // xar v18.2d, v17.2d, v29.2d, #49 // .....................................*.................................................. + // xar v17.2d, v11.2d, v25.2d, #54 // ......................................*................................................. + // xar v11.2d, v9.2d, v27.2d, #44 // ........................................*............................................... + // xar v9.2d, v22.2d, v29.2d, #3 // .........................................*.............................................. + // xar v22.2d, v14.2d, v27.2d, #25 // ..........................................*............................................. + // xar v14.2d, v20.2d, v26.2d, #46 // ...........................................*............................................ + // xar v20.2d, v4.2d, v27.2d, #37 // .............................................*.......................................... + // xar v4.2d, v24.2d, v27.2d, #50 // ..............................................*......................................... + // xar v24.2d, v21.2d, v25.2d, #62 // ...............................................*........................................ + // xar v21.2d, v5.2d, v26.2d, #28 // ................................................*....................................... + // xar v27.2d, v6.2d, v25.2d, #20 // ..................................................*..................................... + // ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ....................................................................................*... + // ld1r {v28.2d}, [x30], #8 // ....................................................................................*... + // str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .....................................................................................*.. + // bcax v5.16b, v10.16b, v7.16b, v11.16b // ...................................................*.................................... + // bcax v6.16b, v11.16b, v8.16b, v7.16b // ....................................................*................................... + // bcax v7.16b, v7.16b, v9.16b, v8.16b // ......................................................*................................. + // bcax v8.16b, v8.16b, v10.16b, v9.16b // .......................................................*................................ + // bcax v9.16b, v9.16b, v11.16b, v10.16b // ........................................................*............................... + // bcax v10.16b, v15.16b, v12.16b, v16.16b // .........................................................*.............................. + // bcax v11.16b, v16.16b, v13.16b, v12.16b // ...........................................................*............................ + // bcax v12.16b, v12.16b, v14.16b, v13.16b // ............................................................*........................... + // bcax v13.16b, v13.16b, v15.16b, v14.16b // .............................................................*.......................... + // bcax v14.16b, v14.16b, v16.16b, v15.16b // ..............................................................*......................... + // bcax v15.16b, v20.16b, v17.16b, v21.16b // ................................................................*....................... + // bcax v16.16b, v21.16b, v18.16b, v17.16b // .................................................................*...................... + // bcax v17.16b, v17.16b, v19.16b, v18.16b // ..................................................................*..................... + // bcax v18.16b, v18.16b, v20.16b, v19.16b // ....................................................................*................... + // bcax v19.16b, v19.16b, v21.16b, v20.16b // .....................................................................*.................. + // bcax v20.16b, v0.16b, v22.16b, v1.16b // ......................................................................*................. + // bcax v21.16b, v1.16b, v23.16b, v22.16b // .......................................................................*................ + // bcax v22.16b, v22.16b, v24.16b, v23.16b // .........................................................................*.............. + // bcax v23.16b, v23.16b, v0.16b, v24.16b // ..........................................................................*............. + // bcax v24.16b, v24.16b, v1.16b, v0.16b // ...........................................................................*............ + // bcax v0.16b, v30.16b, v2.16b, v27.16b // .............................................................................*.......... + // bcax v1.16b, v27.16b, v3.16b, v2.16b // ..............................................................................*......... + // bcax v2.16b, v2.16b, v4.16b, v3.16b // ...............................................................................*........ + // bcax v3.16b, v3.16b, v30.16b, v4.16b // ................................................................................*....... + // bcax v4.16b, v4.16b, v27.16b, v30.16b // ..................................................................................*..... + // eor v0.16b, v0.16b, v28.16b // .......................................................................................* + + mld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + // Instructions: 181 + // Expected cycles: 90 + // Expected IPC: 2.01 + // + // ----------------------------------- cycle (expected) ------------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|-------------- + eor x0, x15, x11, ror #52 // *......................................................................................... // @slothy:interleaving_class=0 + eor x0, x0, x13, ror #48 // *......................................................................................... // @slothy:interleaving_class=0 + eor3 v30.16B, v0.16B, v5.16B, v10.16B // .*........................................................................................ // @slothy:interleaving_class=1 + eor3 v30.16B, v30.16B, v15.16B, v20.16B // .*........................................................................................ // @slothy:interleaving_class=1 + eor x26, x8, x9, ror #57 // ..*....................................................................................... // @slothy:interleaving_class=0 + eor x27, x0, x14, ror #10 // ..*....................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v1.16B, v6.16B, v11.16B // ...*...................................................................................... // @slothy:interleaving_class=1 + eor x29, x16, x28, ror #63 // ...*...................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v29.16B, v16.16B, v21.16B // ....*..................................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x6, ror #51 // ....*..................................................................................... // @slothy:interleaving_class=0 + eor x30, x23, x22, ror #50 // .....*.................................................................................... // @slothy:interleaving_class=0 + eor3 v28.16B, v2.16B, v7.16B, v12.16B // .....*.................................................................................... // @slothy:interleaving_class=1 + eor x0, x26, x10, ror #31 // ......*................................................................................... // @slothy:interleaving_class=0 + eor x29, x29, x19, ror #37 // ......*................................................................................... // @slothy:interleaving_class=0 + eor3 v28.16B, v28.16B, v17.16B, v22.16B // .......*.................................................................................. // @slothy:interleaving_class=1 + eor x27, x27, x12, ror #5 // .......*.................................................................................. // @slothy:interleaving_class=0 + eor3 v27.16B, v3.16B, v8.16B, v13.16B // ........*................................................................................. // @slothy:interleaving_class=1 + eor x30, x30, x24, ror #34 // ........*................................................................................. // @slothy:interleaving_class=0 + eor x0, x0, x7, ror #27 // .........*................................................................................ // @slothy:interleaving_class=0 + eor3 v27.16B, v27.16B, v18.16B, v23.16B // .........*................................................................................ // @slothy:interleaving_class=1 + eor x26, x30, x21, ror #26 // ..........*............................................................................... // @slothy:interleaving_class=0 + eor3 v26.16B, v4.16B, v9.16B, v14.16B // ..........*............................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x25, ror #15 // ...........*.............................................................................. // @slothy:interleaving_class=0 + ror x30, x27, #62 // ...........*.............................................................................. // @slothy:interleaving_class=0 + eor3 v26.16B, v26.16B, v19.16B, v24.16B // ............*............................................................................. // @slothy:interleaving_class=1 + eor x30, x30, x26, ror #57 // ............*............................................................................. // @slothy:interleaving_class=0 + ror x26, x26, #58 // .............*............................................................................ // @slothy:interleaving_class=0 + rax1 v25.2D, v30.2D, v28.2D // .............*............................................................................ // @slothy:interleaving_class=1 + eor x16, x30, x16 // ..............*........................................................................... // @slothy:interleaving_class=0 + rax1 v28.2D, v28.2D, v26.2D // ..............*........................................................................... // @slothy:interleaving_class=1 + eor x28, x30, x28, ror #63 // ...............*.......................................................................... // @slothy:interleaving_class=0 + str x28, [sp, #STACK_LOC_0] // ...............*.......................................................................... // @slothy:writes=stack_0 // @slothy:interleaving_class=0 + rax1 v26.2D, v26.2D, v29.2D // ................*......................................................................... // @slothy:interleaving_class=1 + eor x29, x29, x17, ror #36 // ................*......................................................................... // @slothy:interleaving_class=0 + eor x28, x1, x2, ror #61 // .................*........................................................................ // @slothy:interleaving_class=0 + rax1 v29.2D, v29.2D, v27.2D // .................*........................................................................ // @slothy:interleaving_class=1 + eor x19, x30, x19, ror #37 // ..................*....................................................................... // @slothy:interleaving_class=0 + rax1 v27.2D, v27.2D, v30.2D // ..................*....................................................................... // @slothy:interleaving_class=1 + eor x29, x29, x20, ror #2 // ...................*...................................................................... // @slothy:interleaving_class=0 + eor x28, x28, x4, ror #54 // ...................*...................................................................... // @slothy:interleaving_class=0 + eor v30.16B, v0.16B, v26.16B // ....................*..................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x0, ror #55 // ....................*..................................................................... // @slothy:interleaving_class=0 + xar v0.2D, v2.2D, v29.2D, #2 // .....................*.................................................................... // @slothy:interleaving_class=1 + eor x28, x28, x3, ror #39 // .....................*.................................................................... // @slothy:interleaving_class=0 + eor x28, x28, x5, ror #25 // ......................*................................................................... // @slothy:interleaving_class=0 + xar v2.2D, v12.2D, v29.2D, #21 // ......................*................................................................... // @slothy:interleaving_class=1 + ror x0, x0, #56 // .......................*.................................................................. // @slothy:interleaving_class=0 + eor x0, x0, x29, ror #63 // .......................*.................................................................. // @slothy:interleaving_class=0 + xar v12.2D, v13.2D, v28.2D, #39 // ........................*................................................................. // @slothy:interleaving_class=1 + eor x27, x28, x27, ror #61 // ........................*................................................................. // @slothy:interleaving_class=0 + xar v13.2D, v19.2D, v27.2D, #56 // .........................*................................................................ // @slothy:interleaving_class=1 + eor x13, x0, x13, ror #46 // .........................*................................................................ // @slothy:interleaving_class=0 + eor x28, x29, x28, ror #63 // ..........................*............................................................... // @slothy:interleaving_class=0 + xar v19.2D, v23.2D, v28.2D, #8 // ..........................*............................................................... // @slothy:interleaving_class=1 + eor x29, x30, x20, ror #2 // ...........................*.............................................................. // @slothy:interleaving_class=0 + eor x20, x26, x3, ror #39 // ...........................*.............................................................. // @slothy:interleaving_class=0 + xar v23.2D, v15.2D, v26.2D, #23 // ............................*............................................................. // @slothy:interleaving_class=1 + eor x11, x0, x11, ror #50 // ............................*............................................................. // @slothy:interleaving_class=0 + xar v15.2D, v1.2D, v25.2D, #63 // .............................*............................................................ // @slothy:interleaving_class=1 + eor x25, x28, x25, ror #9 // .............................*............................................................ // @slothy:interleaving_class=0 + eor x3, x28, x21, ror #20 // ..............................*........................................................... // @slothy:interleaving_class=0 + xar v1.2D, v8.2D, v28.2D, #9 // ..............................*........................................................... // @slothy:interleaving_class=1 + eor x21, x26, x1 // ...............................*.......................................................... // @slothy:interleaving_class=0 + xar v8.2D, v16.2D, v25.2D, #19 // ...............................*.......................................................... // @slothy:interleaving_class=1 + eor x9, x27, x9, ror #49 // ................................*......................................................... // @slothy:interleaving_class=0 + eor x24, x28, x24, ror #28 // ................................*......................................................... // @slothy:interleaving_class=0 + xar v16.2D, v7.2D, v29.2D, #58 // .................................*........................................................ // @slothy:interleaving_class=1 + eor x1, x30, x17, ror #36 // .................................*........................................................ // @slothy:interleaving_class=0 + eor x14, x0, x14, ror #8 // ..................................*....................................................... // @slothy:interleaving_class=0 + xar v7.2D, v10.2D, v26.2D, #61 // ..................................*....................................................... // @slothy:interleaving_class=1 + eor x22, x28, x22, ror #44 // ...................................*...................................................... // @slothy:interleaving_class=0 + xar v10.2D, v3.2D, v28.2D, #36 // ...................................*...................................................... // @slothy:interleaving_class=1 + eor x8, x27, x8, ror #56 // ....................................*..................................................... // @slothy:interleaving_class=0 + eor x17, x27, x7, ror #19 // ....................................*..................................................... // @slothy:interleaving_class=0 + xar v3.2D, v18.2D, v28.2D, #43 // .....................................*.................................................... // @slothy:interleaving_class=1 + eor x15, x0, x15, ror #62 // .....................................*.................................................... // @slothy:interleaving_class=0 + bic x7, x20, x22, ror #47 // ......................................*................................................... // @slothy:interleaving_class=0 + xar v18.2D, v17.2D, v29.2D, #49 // ......................................*................................................... // @slothy:interleaving_class=1 + eor x4, x26, x4, ror #54 // .......................................*.................................................. // @slothy:interleaving_class=0 + xar v17.2D, v11.2D, v25.2D, #54 // .......................................*.................................................. // @slothy:interleaving_class=1 + eor x0, x0, x12, ror #3 // ........................................*................................................. // @slothy:interleaving_class=0 + eor x28, x28, x23, ror #58 // ........................................*................................................. // @slothy:interleaving_class=0 + xar v11.2D, v9.2D, v27.2D, #44 // .........................................*................................................ // @slothy:interleaving_class=1 + eor x23, x26, x2, ror #61 // .........................................*................................................ // @slothy:interleaving_class=0 + xar v9.2D, v22.2D, v29.2D, #3 // ..........................................*............................................... // @slothy:interleaving_class=1 + eor x26, x26, x5, ror #25 // ..........................................*............................................... // @slothy:interleaving_class=0 + eor x2, x7, x16, ror #39 // ...........................................*.............................................. // @slothy:interleaving_class=0 + xar v22.2D, v14.2D, v27.2D, #25 // ...........................................*.............................................. // @slothy:interleaving_class=1 + bic x7, x9, x20, ror #42 // ............................................*............................................. // @slothy:interleaving_class=0 + bic x30, x15, x9, ror #16 // ............................................*............................................. // @slothy:interleaving_class=0 + xar v14.2D, v20.2D, v26.2D, #46 // .............................................*............................................ // @slothy:interleaving_class=1 + eor x7, x7, x22, ror #25 // .............................................*............................................ // @slothy:interleaving_class=0 + xar v20.2D, v4.2D, v27.2D, #37 // ..............................................*........................................... // @slothy:interleaving_class=1 + eor x12, x30, x20, ror #58 // ..............................................*........................................... // @slothy:interleaving_class=0 + bic x20, x22, x16, ror #56 // ...............................................*.......................................... // @slothy:interleaving_class=0 + xar v4.2D, v24.2D, v27.2D, #50 // ...............................................*.......................................... // @slothy:interleaving_class=1 + eor x30, x27, x6, ror #43 // ................................................*......................................... // @slothy:interleaving_class=0 + eor x22, x20, x15, ror #23 // ................................................*......................................... // @slothy:interleaving_class=0 + xar v24.2D, v21.2D, v25.2D, #62 // .................................................*........................................ // @slothy:interleaving_class=1 + bic x6, x19, x13, ror #42 // .................................................*........................................ // @slothy:interleaving_class=0 + xar v21.2D, v5.2D, v26.2D, #28 // ..................................................*....................................... // @slothy:interleaving_class=1 + eor x6, x6, x17, ror #41 // ..................................................*....................................... // @slothy:interleaving_class=0 + bic x5, x13, x17, ror #63 // ...................................................*...................................... // @slothy:interleaving_class=0 + xar v27.2D, v6.2D, v25.2D, #20 // ...................................................*...................................... // @slothy:interleaving_class=1 + eor x5, x21, x5, ror #21 // ....................................................*..................................... // @slothy:interleaving_class=0 + bcax v5.16B, v10.16B, v7.16B, v11.16B // ....................................................*..................................... // @slothy:interleaving_class=1 + bic x17, x17, x21, ror #44 // .....................................................*.................................... // @slothy:interleaving_class=0 + eor x27, x27, x10, ror #23 // .....................................................*.................................... // @slothy:interleaving_class=0 + bcax v6.16B, v11.16B, v8.16B, v7.16B // ......................................................*................................... // @slothy:interleaving_class=1 + bic x21, x21, x25, ror #50 // ......................................................*................................... // @slothy:interleaving_class=0 + bic x20, x27, x4, ror #25 // .......................................................*.................................. // @slothy:interleaving_class=0 + bcax v7.16B, v7.16B, v9.16B, v8.16B // .......................................................*.................................. // @slothy:interleaving_class=1 + bic x10, x16, x15, ror #31 // ........................................................*................................. // @slothy:interleaving_class=0 + bcax v8.16B, v8.16B, v10.16B, v9.16B // ........................................................*................................. // @slothy:interleaving_class=1 + eor x16, x21, x19, ror #43 // .........................................................*................................ // @slothy:interleaving_class=0 + eor x21, x17, x25, ror #30 // .........................................................*................................ // @slothy:interleaving_class=0 + bcax v9.16B, v9.16B, v11.16B, v10.16B // ..........................................................*............................... // @slothy:interleaving_class=1 + bic x19, x25, x19, ror #57 // ..........................................................*............................... // @slothy:interleaving_class=0 + ldr x25, [sp, #STACK_OFFSET_COUNT] // ...........................................................*.............................. // @slothy:reads=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v10.16B, v15.16B, v12.16B, v16.16B // ...........................................................*.............................. // @slothy:interleaving_class=1 + eor x17, x10, x9, ror #47 // ............................................................*............................. // @slothy:interleaving_class=0 + bcax v11.16B, v16.16B, v13.16B, v12.16B // ............................................................*............................. // @slothy:interleaving_class=1 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // .............................................................*............................ // @slothy:interleaving_class=0 + eor x15, x20, x28, ror #27 // .............................................................*............................ // @slothy:interleaving_class=0 + bcax v12.16B, v12.16B, v14.16B, v13.16B // ..............................................................*........................... // @slothy:interleaving_class=1 + bic x20, x4, x28, ror #2 // ..............................................................*........................... // @slothy:interleaving_class=0 + bcax v13.16B, v13.16B, v15.16B, v14.16B // ...............................................................*.......................... // @slothy:interleaving_class=1 + eor x10, x20, x1, ror #50 // ...............................................................*.......................... // @slothy:interleaving_class=0 + bic x20, x11, x27, ror #60 // ................................................................*......................... // @slothy:interleaving_class=0 + bcax v14.16B, v14.16B, v16.16B, v15.16B // ................................................................*......................... // @slothy:interleaving_class=1 + eor x20, x20, x4, ror #21 // .................................................................*........................ // @slothy:interleaving_class=0 + bic x4, x28, x1, ror #48 // .................................................................*........................ // @slothy:interleaving_class=0 + bcax v15.16B, v20.16B, v17.16B, v21.16B // ..................................................................*....................... // @slothy:interleaving_class=1 + bic x1, x1, x11, ror #57 // ..................................................................*....................... // @slothy:interleaving_class=0 + bcax v16.16B, v21.16B, v18.16B, v17.16B // ...................................................................*...................... // @slothy:interleaving_class=1 + ldr x28, [x9, x25, lsl #3] // ...................................................................*...................... // @slothy:interleaving_class=0 + ldr x9, [sp, #STACK_LOC_0] // ....................................................................*..................... // @slothy:reads=stack_0 // @slothy:interleaving_class=0 + bcax v17.16B, v17.16B, v19.16B, v18.16B // ....................................................................*..................... // @slothy:interleaving_class=1 + add x25, x25, #1 // .....................................................................*.................... // @slothy:interleaving_class=0 + str x25, [sp, #STACK_OFFSET_COUNT] // .....................................................................*.................... // @slothy:writes=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v18.16B, v18.16B, v20.16B, v19.16B // ......................................................................*................... // @slothy:interleaving_class=1 + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ......................................................................*................... // @slothy:ignore_useless_output // @slothy:interleaving_class=0 + bcax v19.16B, v19.16B, v21.16B, v20.16B // .......................................................................*.................. // @slothy:interleaving_class=1 + eor x25, x1, x27, ror #53 // .......................................................................*.................. // @slothy:interleaving_class=0 + bic x27, x30, x26, ror #47 // ........................................................................*................. // @slothy:interleaving_class=0 + bcax v20.16B, v0.16B, v22.16B, v1.16B // ........................................................................*................. // @slothy:interleaving_class=1 + eor x1, x5, x28 // .........................................................................*................ // @slothy:interleaving_class=0 + bcax v21.16B, v1.16B, v23.16B, v22.16B // .........................................................................*................ // @slothy:interleaving_class=1 + eor x5, x4, x11, ror #41 // ..........................................................................*............... // @slothy:interleaving_class=0 + eor x11, x19, x13, ror #35 // ..........................................................................*............... // @slothy:interleaving_class=0 + bcax v22.16B, v22.16B, v24.16B, v23.16B // ...........................................................................*.............. // @slothy:interleaving_class=1 + bic x13, x26, x24, ror #10 // ...........................................................................*.............. // @slothy:interleaving_class=0 + eor x28, x27, x24, ror #57 // ............................................................................*............. // @slothy:interleaving_class=0 + bcax v23.16B, v23.16B, v0.16B, v24.16B // ............................................................................*............. // @slothy:interleaving_class=1 + bic x27, x24, x9, ror #47 // .............................................................................*............ // @slothy:interleaving_class=0 + bcax v24.16B, v24.16B, v1.16B, v0.16B // .............................................................................*............ // @slothy:interleaving_class=1 + bic x19, x23, x3, ror #9 // ..............................................................................*........... // @slothy:interleaving_class=0 + bic x4, x29, x14, ror #41 // ..............................................................................*........... // @slothy:interleaving_class=0 + bcax v0.16B, v30.16B, v2.16B, v27.16B // ...............................................................................*.......... // @slothy:interleaving_class=1 + eor x24, x19, x29, ror #44 // ...............................................................................*.......... // @slothy:interleaving_class=0 + bic x29, x3, x29, ror #35 // ................................................................................*......... // @slothy:interleaving_class=0 + bcax v1.16B, v27.16B, v3.16B, v2.16B // ................................................................................*......... // @slothy:interleaving_class=1 + eor x13, x13, x9, ror #57 // .................................................................................*........ // @slothy:interleaving_class=0 + bcax v2.16B, v2.16B, v4.16B, v3.16B // .................................................................................*........ // @slothy:interleaving_class=1 + eor x19, x29, x14, ror #12 // ..................................................................................*....... // @slothy:interleaving_class=0 + bic x29, x9, x0, ror #19 // ..................................................................................*....... // @slothy:interleaving_class=0 + bcax v3.16B, v3.16B, v30.16B, v4.16B // ...................................................................................*...... // @slothy:interleaving_class=1 + bic x14, x14, x8, ror #5 // ...................................................................................*...... // @slothy:interleaving_class=0 + bcax v4.16B, v4.16B, v27.16B, v30.16B // ....................................................................................*..... // @slothy:interleaving_class=1 + eor x9, x14, x23, ror #43 // ....................................................................................*..... // @slothy:interleaving_class=0 + eor x14, x4, x8, ror #46 // .....................................................................................*.... // @slothy:interleaving_class=0 + bic x23, x8, x23, ror #38 // .....................................................................................*.... // @slothy:interleaving_class=0 + eor x8, x27, x0, ror #2 // ......................................................................................*... // @slothy:interleaving_class=0 + eor x4, x23, x3, ror #47 // ......................................................................................*... // @slothy:interleaving_class=0 + bic x3, x0, x30, ror #5 // .......................................................................................*.. // @slothy:interleaving_class=0 + eor x23, x3, x26, ror #52 // .......................................................................................*.. // @slothy:interleaving_class=0 + eor x3, x29, x30, ror #24 // ........................................................................................*. // @slothy:interleaving_class=0 + ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ........................................................................................*. // @slothy:reads=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + ld1r {v28.2D}, [x30], #8 // .........................................................................................* // @slothy:interleaving_class=1 + str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .........................................................................................* // @slothy:writes=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + eor v0.16B, v0.16B, v28.16B // ..........................................................................................* // @slothy:interleaving_class=1 + + // ------------------------------------ cycle (expected) ------------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|--------------- + // eor x0, x15, x11, ror #52 // *.......................................................................................... + // eor x0, x0, x13, ror #48 // *.......................................................................................... + // eor x26, x8, x9, ror #57 // ..*........................................................................................ + // eor x27, x0, x14, ror #10 // ..*........................................................................................ + // eor x29, x16, x28, ror #63 // ...*....................................................................................... + // eor x26, x26, x6, ror #51 // ....*...................................................................................... + // eor x30, x23, x22, ror #50 // .....*..................................................................................... + // eor x0, x26, x10, ror #31 // ......*.................................................................................... + // eor x29, x29, x19, ror #37 // ......*.................................................................................... + // eor x27, x27, x12, ror #5 // .......*................................................................................... + // eor x30, x30, x24, ror #34 // ........*.................................................................................. + // eor x0, x0, x7, ror #27 // .........*................................................................................. + // eor x26, x30, x21, ror #26 // ..........*................................................................................ + // eor x26, x26, x25, ror #15 // ...........*............................................................................... + // ror x30, x27, #62 // ...........*............................................................................... + // eor x30, x30, x26, ror #57 // ............*.............................................................................. + // ror x26, x26, #58 // .............*............................................................................. + // eor x16, x30, x16 // ..............*............................................................................ + // eor x28, x30, x28, ror #63 // ...............*........................................................................... + // str x28, [sp, #STACK_LOC_0] // ...............*........................................................................... + // eor x29, x29, x17, ror #36 // ................*.......................................................................... + // eor x28, x1, x2, ror #61 // .................*......................................................................... + // eor x19, x30, x19, ror #37 // ..................*........................................................................ + // eor x29, x29, x20, ror #2 // ...................*....................................................................... + // eor x28, x28, x4, ror #54 // ...................*....................................................................... + // eor x26, x26, x0, ror #55 // ....................*...................................................................... + // eor x28, x28, x3, ror #39 // .....................*..................................................................... + // eor x28, x28, x5, ror #25 // ......................*.................................................................... + // ror x0, x0, #56 // .......................*................................................................... + // eor x0, x0, x29, ror #63 // .......................*................................................................... + // eor x27, x28, x27, ror #61 // ........................*.................................................................. + // eor x13, x0, x13, ror #46 // .........................*................................................................. + // eor x28, x29, x28, ror #63 // ..........................*................................................................ + // eor x29, x30, x20, ror #2 // ...........................*............................................................... + // eor x20, x26, x3, ror #39 // ...........................*............................................................... + // eor x11, x0, x11, ror #50 // ............................*.............................................................. + // eor x25, x28, x25, ror #9 // .............................*............................................................. + // eor x3, x28, x21, ror #20 // ..............................*............................................................ + // eor x21, x26, x1 // ...............................*........................................................... + // eor x9, x27, x9, ror #49 // ................................*.......................................................... + // eor x24, x28, x24, ror #28 // ................................*.......................................................... + // eor x1, x30, x17, ror #36 // .................................*......................................................... + // eor x14, x0, x14, ror #8 // ..................................*........................................................ + // eor x22, x28, x22, ror #44 // ...................................*....................................................... + // eor x8, x27, x8, ror #56 // ....................................*...................................................... + // eor x17, x27, x7, ror #19 // ....................................*...................................................... + // eor x15, x0, x15, ror #62 // .....................................*..................................................... + // bic x7, x20, x22, ror #47 // ......................................*.................................................... + // eor x4, x26, x4, ror #54 // .......................................*................................................... + // eor x0, x0, x12, ror #3 // ........................................*.................................................. + // eor x28, x28, x23, ror #58 // ........................................*.................................................. + // eor x23, x26, x2, ror #61 // .........................................*................................................. + // eor x26, x26, x5, ror #25 // ..........................................*................................................ + // eor x2, x7, x16, ror #39 // ...........................................*............................................... + // bic x7, x9, x20, ror #42 // ............................................*.............................................. + // bic x30, x15, x9, ror #16 // ............................................*.............................................. + // eor x7, x7, x22, ror #25 // .............................................*............................................. + // eor x12, x30, x20, ror #58 // ..............................................*............................................ + // bic x20, x22, x16, ror #56 // ...............................................*........................................... + // eor x30, x27, x6, ror #43 // ................................................*.......................................... + // eor x22, x20, x15, ror #23 // ................................................*.......................................... + // bic x6, x19, x13, ror #42 // .................................................*......................................... + // eor x6, x6, x17, ror #41 // ..................................................*........................................ + // bic x5, x13, x17, ror #63 // ...................................................*....................................... + // eor x5, x21, x5, ror #21 // ....................................................*...................................... + // bic x17, x17, x21, ror #44 // .....................................................*..................................... + // eor x27, x27, x10, ror #23 // .....................................................*..................................... + // bic x21, x21, x25, ror #50 // ......................................................*.................................... + // bic x20, x27, x4, ror #25 // .......................................................*................................... + // bic x10, x16, x15, ror #31 // ........................................................*.................................. + // eor x16, x21, x19, ror #43 // .........................................................*................................. + // eor x21, x17, x25, ror #30 // .........................................................*................................. + // bic x19, x25, x19, ror #57 // ..........................................................*................................ + // ldr x25, [sp, #STACK_OFFSET_COUNT] // ...........................................................*............................... + // eor x17, x10, x9, ror #47 // ............................................................*.............................. + // ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // .............................................................*............................. + // eor x15, x20, x28, ror #27 // .............................................................*............................. + // bic x20, x4, x28, ror #2 // ..............................................................*............................ + // eor x10, x20, x1, ror #50 // ...............................................................*........................... + // bic x20, x11, x27, ror #60 // ................................................................*.......................... + // eor x20, x20, x4, ror #21 // .................................................................*......................... + // bic x4, x28, x1, ror #48 // .................................................................*......................... + // bic x1, x1, x11, ror #57 // ..................................................................*........................ + // ldr x28, [x9, x25, LSL #3] // ...................................................................*....................... + // ldr x9, [sp, #STACK_LOC_0] // ....................................................................*...................... + // add x25, x25, #1 // .....................................................................*..................... + // str x25, [sp, #STACK_OFFSET_COUNT] // .....................................................................*..................... + // cmp x25, #(KECCAK_F1600_ROUNDS-1) // ......................................................................*.................... + // eor x25, x1, x27, ror #53 // .......................................................................*................... + // bic x27, x30, x26, ror #47 // ........................................................................*.................. + // eor x1, x5, x28 // .........................................................................*................. + // eor x5, x4, x11, ror #41 // ..........................................................................*................ + // eor x11, x19, x13, ror #35 // ..........................................................................*................ + // bic x13, x26, x24, ror #10 // ...........................................................................*............... + // eor x28, x27, x24, ror #57 // ............................................................................*.............. + // bic x27, x24, x9, ror #47 // .............................................................................*............. + // bic x19, x23, x3, ror #9 // ..............................................................................*............ + // bic x4, x29, x14, ror #41 // ..............................................................................*............ + // eor x24, x19, x29, ror #44 // ...............................................................................*........... + // bic x29, x3, x29, ror #35 // ................................................................................*.......... + // eor x13, x13, x9, ror #57 // .................................................................................*......... + // eor x19, x29, x14, ror #12 // ..................................................................................*........ + // bic x29, x9, x0, ror #19 // ..................................................................................*........ + // bic x14, x14, x8, ror #5 // ...................................................................................*....... + // eor x9, x14, x23, ror #43 // ....................................................................................*...... + // eor x14, x4, x8, ror #46 // .....................................................................................*..... + // bic x23, x8, x23, ror #38 // .....................................................................................*..... + // eor x8, x27, x0, ror #2 // ......................................................................................*.... + // eor x4, x23, x3, ror #47 // ......................................................................................*.... + // bic x3, x0, x30, ror #5 // .......................................................................................*... + // eor x23, x3, x26, ror #52 // .......................................................................................*... + // eor x3, x29, x30, ror #24 // ........................................................................................*.. + // eor3 v30.16b, v0.16b, v5.16b, v10.16b // .*......................................................................................... + // eor3 v30.16b, v30.16b, v15.16b, v20.16b // .*......................................................................................... + // eor3 v29.16b, v1.16b, v6.16b, v11.16b // ...*....................................................................................... + // eor3 v29.16b, v29.16b, v16.16b, v21.16b // ....*...................................................................................... + // eor3 v28.16b, v2.16b, v7.16b, v12.16b // .....*..................................................................................... + // eor3 v28.16b, v28.16b, v17.16b, v22.16b // .......*................................................................................... + // eor3 v27.16b, v3.16b, v8.16b, v13.16b // ........*.................................................................................. + // eor3 v27.16b, v27.16b, v18.16b, v23.16b // .........*................................................................................. + // eor3 v26.16b, v4.16b, v9.16b, v14.16b // ..........*................................................................................ + // eor3 v26.16b, v26.16b, v19.16b, v24.16b // ............*.............................................................................. + // rax1 v25.2d, v30.2d, v28.2d // .............*............................................................................. + // rax1 v28.2d, v28.2d, v26.2d // ..............*............................................................................ + // rax1 v26.2d, v26.2d, v29.2d // ................*.......................................................................... + // rax1 v29.2d, v29.2d, v27.2d // .................*......................................................................... + // rax1 v27.2d, v27.2d, v30.2d // ..................*........................................................................ + // eor v30.16b, v0.16b, v26.16b // ....................*...................................................................... + // xar v0.2d, v2.2d, v29.2d, #2 // .....................*..................................................................... + // xar v2.2d, v12.2d, v29.2d, #21 // ......................*.................................................................... + // xar v12.2d, v13.2d, v28.2d, #39 // ........................*.................................................................. + // xar v13.2d, v19.2d, v27.2d, #56 // .........................*................................................................. + // xar v19.2d, v23.2d, v28.2d, #8 // ..........................*................................................................ + // xar v23.2d, v15.2d, v26.2d, #23 // ............................*.............................................................. + // xar v15.2d, v1.2d, v25.2d, #63 // .............................*............................................................. + // xar v1.2d, v8.2d, v28.2d, #9 // ..............................*............................................................ + // xar v8.2d, v16.2d, v25.2d, #19 // ...............................*........................................................... + // xar v16.2d, v7.2d, v29.2d, #58 // .................................*......................................................... + // xar v7.2d, v10.2d, v26.2d, #61 // ..................................*........................................................ + // xar v10.2d, v3.2d, v28.2d, #36 // ...................................*....................................................... + // xar v3.2d, v18.2d, v28.2d, #43 // .....................................*..................................................... + // xar v18.2d, v17.2d, v29.2d, #49 // ......................................*.................................................... + // xar v17.2d, v11.2d, v25.2d, #54 // .......................................*................................................... + // xar v11.2d, v9.2d, v27.2d, #44 // .........................................*................................................. + // xar v9.2d, v22.2d, v29.2d, #3 // ..........................................*................................................ + // xar v22.2d, v14.2d, v27.2d, #25 // ...........................................*............................................... + // xar v14.2d, v20.2d, v26.2d, #46 // .............................................*............................................. + // xar v20.2d, v4.2d, v27.2d, #37 // ..............................................*............................................ + // xar v4.2d, v24.2d, v27.2d, #50 // ...............................................*........................................... + // xar v24.2d, v21.2d, v25.2d, #62 // .................................................*......................................... + // xar v21.2d, v5.2d, v26.2d, #28 // ..................................................*........................................ + // xar v27.2d, v6.2d, v25.2d, #20 // ...................................................*....................................... + // ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ........................................................................................*.. + // ld1r {v28.2d}, [x30], #8 // .........................................................................................*. + // str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .........................................................................................*. + // bcax v5.16b, v10.16b, v7.16b, v11.16b // ....................................................*...................................... + // bcax v6.16b, v11.16b, v8.16b, v7.16b // ......................................................*.................................... + // bcax v7.16b, v7.16b, v9.16b, v8.16b // .......................................................*................................... + // bcax v8.16b, v8.16b, v10.16b, v9.16b // ........................................................*.................................. + // bcax v9.16b, v9.16b, v11.16b, v10.16b // ..........................................................*................................ + // bcax v10.16b, v15.16b, v12.16b, v16.16b // ...........................................................*............................... + // bcax v11.16b, v16.16b, v13.16b, v12.16b // ............................................................*.............................. + // bcax v12.16b, v12.16b, v14.16b, v13.16b // ..............................................................*............................ + // bcax v13.16b, v13.16b, v15.16b, v14.16b // ...............................................................*........................... + // bcax v14.16b, v14.16b, v16.16b, v15.16b // ................................................................*.......................... + // bcax v15.16b, v20.16b, v17.16b, v21.16b // ..................................................................*........................ + // bcax v16.16b, v21.16b, v18.16b, v17.16b // ...................................................................*....................... + // bcax v17.16b, v17.16b, v19.16b, v18.16b // ....................................................................*...................... + // bcax v18.16b, v18.16b, v20.16b, v19.16b // ......................................................................*.................... + // bcax v19.16b, v19.16b, v21.16b, v20.16b // .......................................................................*................... + // bcax v20.16b, v0.16b, v22.16b, v1.16b // ........................................................................*.................. + // bcax v21.16b, v1.16b, v23.16b, v22.16b // .........................................................................*................. + // bcax v22.16b, v22.16b, v24.16b, v23.16b // ...........................................................................*............... + // bcax v23.16b, v23.16b, v0.16b, v24.16b // ............................................................................*.............. + // bcax v24.16b, v24.16b, v1.16b, v0.16b // .............................................................................*............. + // bcax v0.16b, v30.16b, v2.16b, v27.16b // ...............................................................................*........... + // bcax v1.16b, v27.16b, v3.16b, v2.16b // ................................................................................*.......... + // bcax v2.16b, v2.16b, v4.16b, v3.16b // .................................................................................*......... + // bcax v3.16b, v3.16b, v30.16b, v4.16b // ...................................................................................*....... + // bcax v4.16b, v4.16b, v27.16b, v30.16b // ....................................................................................*...... + // eor v0.16b, v0.16b, v28.16b // ..........................................................................................* + + mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + + ble mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + final_scalar_rotate + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/* ***************** REGISTER DEALLOCATIONS ****************** */ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64/src/keccakf1600_round_constants.c b/dev/fips202/aarch64/src/keccakf1600_round_constants.c index eb578bcfcd..df0652c1f4 100644 --- a/dev/fips202/aarch64/src/keccakf1600_round_constants.c +++ b/dev/fips202/aarch64/src/keccakf1600_round_constants.c @@ -14,6 +14,7 @@ #if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \ defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \ defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) @@ -34,7 +35,8 @@ MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t #else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED */ @@ -42,6 +44,7 @@ MLD_EMPTY_CU(fips202_aarch64_round_constants) #endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/dev/fips202/aarch64/x3_v84a_scalar.h b/dev/fips202/aarch64/x3_v84a_scalar.h new file mode 100644 index 0000000000..fa7ad121eb --- /dev/null +++ b/dev/fips202/aarch64/x3_v84a_scalar.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H +#define MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X3 +/* Guard for assembly file */ +#define MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || + !mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm( + state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S new file mode 100644 index 0000000000..94c1d3dc47 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S @@ -0,0 +1,527 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// + +/*yaml + Name: keccak_f1600_x1_scalar_symbolic + Description: AArch64 scalar symbolic implementation of Keccak-f[1600] permutation for single state (clean version for optimization) + Signature: void mld_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t *state + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 128 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_rc .req x1 + const_addr .req x26 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + Aba .req x1 + Abe .req x6 + Abi .req x11 + Abo .req x16 + Abu .req x21 + Aga .req x2 + Age .req x7 + Agi .req x12 + Ago .req x17 + Agu .req x22 + Aka .req x3 + Ake .req x8 + Aki .req x13 + Ako .req x28 + Aku .req x23 + Ama .req x4 + Ame .req x9 + Ami .req x14 + Amo .req x19 + Amu .req x24 + Asa .req x5 + Ase .req x10 + Asi .req x15 + Aso .req x20 + Asu .req x25 + +/************************ MACROS ****************************/ + +/* Allocate a large number of stack slots by default so one can quickly + * check functional correctness by stopping SLOTHY's register allocation + * and stack spilling process while still using a large number of spills. + * + * NOTE: + * This should to be manually trimmed to the number of stack locations + * actually used after running SLOTHY. + */ +#define STACK_LOCS 40 + +// GPRs (16*6), count (8), const (8), input (8), padding (8) +#define STACK_SIZE (16*6 + 3*8 + 8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (3*8+8) +#define STACK_LOC_INPUT (0*8) +#define STACK_LOC_CONST (1*8) +#define STACK_LOC_COUNT (2*8) +#define STACK_LOC_MISC (16*6 + 4*8) + +#define STACK_LOC_0 ((STACK_LOC_MISC) + 0*8) +#define STACK_LOC_1 ((STACK_LOC_MISC) + 1*8) +#define STACK_LOC_2 ((STACK_LOC_MISC) + 2*8) +#define STACK_LOC_3 ((STACK_LOC_MISC) + 3*8) +#define STACK_LOC_4 ((STACK_LOC_MISC) + 4*8) +#define STACK_LOC_5 ((STACK_LOC_MISC) + 5*8) +#define STACK_LOC_6 ((STACK_LOC_MISC) + 6*8) +#define STACK_LOC_7 ((STACK_LOC_MISC) + 7*8) +#define STACK_LOC_8 ((STACK_LOC_MISC) + 8*8) +#define STACK_LOC_9 ((STACK_LOC_MISC) + 9*8) +#define STACK_LOC_10 ((STACK_LOC_MISC) + 10*8) +#define STACK_LOC_11 ((STACK_LOC_MISC) + 11*8) +#define STACK_LOC_12 ((STACK_LOC_MISC) + 12*8) +#define STACK_LOC_13 ((STACK_LOC_MISC) + 13*8) +#define STACK_LOC_14 ((STACK_LOC_MISC) + 14*8) +#define STACK_LOC_15 ((STACK_LOC_MISC) + 15*8) +#define STACK_LOC_16 ((STACK_LOC_MISC) + 16*8) +#define STACK_LOC_17 ((STACK_LOC_MISC) + 17*8) +#define STACK_LOC_18 ((STACK_LOC_MISC) + 18*8) +#define STACK_LOC_19 ((STACK_LOC_MISC) + 19*8) +#define STACK_LOC_20 ((STACK_LOC_MISC) + 20*8) +#define STACK_LOC_21 ((STACK_LOC_MISC) + 21*8) +#define STACK_LOC_22 ((STACK_LOC_MISC) + 22*8) +#define STACK_LOC_23 ((STACK_LOC_MISC) + 23*8) +#define STACK_LOC_24 ((STACK_LOC_MISC) + 24*8) +#define STACK_LOC_25 ((STACK_LOC_MISC) + 25*8) +#define STACK_LOC_26 ((STACK_LOC_MISC) + 26*8) +#define STACK_LOC_27 ((STACK_LOC_MISC) + 27*8) +#define STACK_LOC_28 ((STACK_LOC_MISC) + 28*8) +#define STACK_LOC_29 ((STACK_LOC_MISC) + 29*8) +#define STACK_LOC_30 ((STACK_LOC_MISC) + 30*8) +#define STACK_LOC_31 ((STACK_LOC_MISC) + 31*8) +#define STACK_LOC_32 ((STACK_LOC_MISC) + 32*8) +#define STACK_LOC_33 ((STACK_LOC_MISC) + 33*8) +#define STACK_LOC_34 ((STACK_LOC_MISC) + 34*8) +#define STACK_LOC_35 ((STACK_LOC_MISC) + 35*8) +#define STACK_LOC_36 ((STACK_LOC_MISC) + 36*8) +#define STACK_LOC_37 ((STACK_LOC_MISC) + 37*8) +#define STACK_LOC_38 ((STACK_LOC_MISC) + 38*8) +#define STACK_LOC_39 ((STACK_LOC_MISC) + 39*8) + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +.macro keccak_f1600_round_initial + eor5 X, Ama, Asa, Aba, Aga, Aka + eor5 X, Ame, Ase, Abe, Age, Ake + eor5 X, Ami, Asi, Abi, Agi, Aki + eor5 X, Amo, Aso, Abo, Ago, Ako + eor5 X, Amu, Asu, Abu, Agu, Aku + + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + + eor X, Aba, X + eor X, Abi, X + eor X, Aki, X + eor X, Ako, X + eor X, Amu, X + eor X, Aso, X + eor X, Ama, X + eor X, Abe, X + eor X, Ago, X + eor X, Ame, X + eor X, Agi, X + eor X, Aka, X + eor X, Abo, X + eor X, Amo, X + eor X, Ami, X + eor X, Ake, X + eor X, Agu, X + eor X, Asi, X + eor X, Aku, X + eor X, Asa, X + eor X, Abu, X + eor X, Asu, X + eor X, Ase, X + eor X, Aga, X + eor X, Age, X + + ldr X, [sp, #STACK_LOC_CONST] + ldr X, [X] + mov X, #1 + str X, [sp, #STACK_LOC_COUNT] // @slothy:writes=STACK_LOC_COUNT + + chi_step_ror Aga, X, X, X, 47, 39 + chi_step_ror Age, X, X, X, 42, 25 + chi_step_ror Agi, X, X, X, 16, 58 + chi_step_ror Ago, X, X, X, 31, 47 + chi_step_ror Agu, X, X, X, 56, 23 + chi_step_ror Aka, X, X, X, 19, 24 + chi_step_ror Ake, X, X, X, 47, 2 + chi_step_ror Aki, X, X, X, 10, 57 + chi_step_ror Ako, X, X, X, 47, 57 + chi_step_ror Aku, X, X, X, 5, 52 + chi_step_ror Ama, X, X, X, 38, 47 + chi_step_ror Ame, X, X, X, 5, 43 + chi_step_ror Ami, X, X, X, 41, 46 + chi_step_ror Amo, X, X, X, 35, 12 + chi_step_ror Amu, X, X, X, 9, 44 + chi_step_ror Asa, X, X, X, 48, 41 + chi_step_ror Ase, X, X, X, 2, 50 + chi_step_ror Asi, X, X, X, 25, 27 + chi_step_ror Aso, X, X, X, 60, 21 + chi_step_ror Asu, X, X, X, 57, 53 + chi_step_ror2 Aba, X, X, X, 63, 21 + chi_step_ror Abe, X, X, X, 42, 41 + chi_step_ror Abi, X, X, X, 57, 35 + chi_step_ror Abo, X, X, X, 50, 43 + chi_step_ror Abu, X, X, X, 44, 30 + + eor Aba, Aba, X + +.endm + +.macro keccak_f1600_round_noninitial + + eor X, Aba, Aga, ror #61 + eor X, X, Ama, ror #54 + eor X, X, Aka, ror #39 + eor X, X, Asa, ror #25 + + eor X, Ake, Ame, ror #57 + eor X, X, Abe, ror #51 + eor X, X, Ase, ror #31 + eor X, X, Age, ror #27 + + eor X, Asi, Abi, ror #52 + eor X, X, Aki, ror #48 + eor X, X, Ami, ror #10 + eor X, X, Agi, ror #5 + + eor X, Abo, Ako, ror #63 + eor X, X, Amo, ror #37 + eor X, X, Ago, ror #36 + eor X, X, Aso, ror #2 + + eor X, Aku, Agu, ror #50 + eor X, X, Amu, ror #34 + eor X, X, Abu, ror #26 + eor X, X, Asu, ror #15 + + eor X, X, X, ror #61 + ror X, X, #62 + eor X, X, X, ror #57 + ror X, X, #58 + eor X, X, X, ror #55 + ror X, X, #56 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + + eor X, X, Aba + eor X, X, Abi, ror #50 + eor X, X, Aki, ror #46 + eor X, X, Ako, ror #63 + eor X, X, Amu, ror #28 + eor X, X, Aso, ror #2 + eor X, X, Ama, ror #54 + eor X, X, Abe, ror #43 + eor X, X, Ago, ror #36 + eor X, X, Ame, ror #49 + eor X, X, Agi, ror #3 + eor X, X, Aka, ror #39 + eor X, X, Abo + eor X, X, Amo, ror #37 + eor X, X, Ami, ror #8 + eor X, X, Ake, ror #56 + eor X, X, Agu, ror #44 + eor X, X, Asi, ror #62 + eor X, X, Aku, ror #58 + eor X, X, Asa, ror #25 + eor X, X, Abu, ror #20 + eor X, X, Asu, ror #9 + eor X, X, Ase, ror #23 + eor X, X, Aga, ror #61 + eor X, X, Age, ror #19 + + ldr X, [sp, #STACK_LOC_CONST] + ldr X, [sp, #STACK_LOC_COUNT] // @slothy:reads=STACK_LOC_COUNT + ldr X, [X, X, LSL #3] + add X, X, #1 + cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + str X, [sp, #STACK_LOC_COUNT] // @slothy:writes=STACK_LOC_COUNT + + chi_step_ror Aga, X, X, X, 47, 39 + chi_step_ror Age, X, X, X, 42, 25 + chi_step_ror Agi, X, X, X, 16, 58 + chi_step_ror Ago, X, X, X, 31, 47 + chi_step_ror Agu, X, X, X, 56, 23 + chi_step_ror Aka, X, X, X, 19, 24 + chi_step_ror Ake, X, X, X, 47, 2 + chi_step_ror Aki, X, X, X, 10, 57 + chi_step_ror Ako, X, X, X, 47, 57 + chi_step_ror Aku, X, X, X, 5, 52 + chi_step_ror Ama, X, X, X, 38, 47 + chi_step_ror Ame, X, X, X, 5, 43 + chi_step_ror Ami, X, X, X, 41, 46 + chi_step_ror Amo, X, X, X, 35, 12 + chi_step_ror Amu, X, X, X, 9, 44 + chi_step_ror Asa, X, X, X, 48, 41 + chi_step_ror Ase, X, X, X, 2, 50 + chi_step_ror Asi, X, X, X, 25, 27 + chi_step_ror Aso, X, X, X, 60, 21 + chi_step_ror Asu, X, X, X, 57, 53 + chi_step_ror2 Aba, X, X, X, 63, 21 + chi_step_ror Abe, X, X, X, 42, 41 + chi_step_ror Abi, X, X, X, 57, 35 + chi_step_ror Abo, X, X, X, 50, 43 + chi_step_ror Abu, X, X, X, 44, 30 + + eor Aba, Aba, X +.endm + +.macro load_state + ldp Aba, Abe, [input_addr, #(1*8*0)] + ldp Abi, Abo, [input_addr, #(1*8*2)] + ldp Abu, Aga, [input_addr, #(1*8*4)] + ldp Age, Agi, [input_addr, #(1*8*6)] + ldp Ago, Agu, [input_addr, #(1*8*8)] + ldp Aka, Ake, [input_addr, #(1*8*10)] + ldp Aki, Ako, [input_addr, #(1*8*12)] + ldp Aku, Ama, [input_addr, #(1*8*14)] + ldp Ame, Ami, [input_addr, #(1*8*16)] + ldp Amo, Amu, [input_addr, #(1*8*18)] + ldp Asa, Ase, [input_addr, #(1*8*20)] + ldp Asi, Aso, [input_addr, #(1*8*22)] + ldr Asu, [input_addr, #(1*8*24)] +.endm + +.macro store_state + stp Aba, Abe, [input_addr, #(1*8*0)] + stp Abi, Abo, [input_addr, #(1*8*2)] + stp Abu, Aga, [input_addr, #(1*8*4)] + stp Age, Agi, [input_addr, #(1*8*6)] + stp Ago, Agu, [input_addr, #(1*8*8)] + stp Aka, Ake, [input_addr, #(1*8*10)] + stp Aki, Ako, [input_addr, #(1*8*12)] + stp Aku, Ama, [input_addr, #(1*8*14)] + stp Ame, Ami, [input_addr, #(1*8*16)] + stp Amo, Amu, [input_addr, #(1*8*18)] + stp Asa, Ase, [input_addr, #(1*8*20)] + stp Asi, Aso, [input_addr, #(1*8*22)] + str Asu, [input_addr, #(1*8*24)] +.endm + +.macro final_rotate + ror Abe, Abe,#(64-21) + ror Abi, Abi,#(64-14) + ror Abu, Abu,#(64-44) + ror Aga, Aga,#(64-3) + ror Age, Age,#(64-45) + ror Agi, Agi,#(64-61) + ror Ago, Ago,#(64-28) + ror Agu, Agu,#(64-20) + ror Aka, Aka,#(64-25) + ror Ake, Ake,#(64-8) + ror Aki, Aki,#(64-18) + ror Ako, Ako,#(64-1) + ror Aku, Aku,#(64-6) + ror Ama, Ama,#(64-10) + ror Ame, Ame,#(64-15) + ror Ami, Ami,#(64-56) + ror Amo, Amo,#(64-27) + ror Amu, Amu,#(64-36) + ror Asa, Asa,#(64-39) + ror Ase, Ase,#(64-41) + ror Asi, Asi,#(64-2) + ror Aso, Aso,#(64-62) + ror Asu, Asu,#(64-55) +.endm + +#define KECCAK_F1600_ROUNDS 24 + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm) + alloc_stack + save_gprs + +mld_keccak_f1600_x1_scalar_initial: + mov const_addr, input_rc + str input_rc, [sp, #STACK_LOC_CONST] + load_state + str input_addr, [sp, #STACK_LOC_INPUT] // @slothy:writes=STACK_LOC_INPUT + +mld_keccak_f1600_x1_scalar_initial_start: + keccak_f1600_round_initial + +mld_keccak_f1600_x1_scalar_loop: + keccak_f1600_round_noninitial +mld_keccak_f1600_x1_scalar_end_loop: + ble mld_keccak_f1600_x1_scalar_loop + +mld_keccak_f1600_x1_scalar_final: + final_rotate + ldr input_addr, [sp, #STACK_LOC_INPUT] // @slothy:reads=STACK_LOC_INPUT + store_state +mld_keccak_f1600_x1_scalar_end_final: + + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_rc + .unreq const_addr + .unreq Aba + .unreq Abe + .unreq Abi + .unreq Abo + .unreq Abu + .unreq Aga + .unreq Age + .unreq Agi + .unreq Ago + .unreq Agu + .unreq Aka + .unreq Ake + .unreq Aki + .unreq Ako + .unreq Aku + .unreq Ama + .unreq Ame + .unreq Ami + .unreq Amo + .unreq Amu + .unreq Asa + .unreq Ase + .unreq Asi + .unreq Aso + .unreq Asu + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_LOC_INPUT +#undef STACK_LOC_CONST +#undef STACK_LOC_COUNT +#undef STACK_LOC_MISC +#undef STACK_LOC_0 +#undef STACK_LOC_1 +#undef STACK_LOC_2 +#undef STACK_LOC_3 +#undef STACK_LOC_4 +#undef STACK_LOC_5 +#undef STACK_LOC_6 +#undef STACK_LOC_7 +#undef STACK_LOC_8 +#undef STACK_LOC_9 +#undef STACK_LOC_10 +#undef STACK_LOC_11 +#undef STACK_LOC_12 +#undef STACK_LOC_13 +#undef STACK_LOC_14 +#undef STACK_LOC_15 +#undef STACK_LOC_16 +#undef STACK_LOC_17 +#undef STACK_LOC_18 +#undef STACK_LOC_19 +#undef STACK_LOC_20 +#undef STACK_LOC_21 +#undef STACK_LOC_22 +#undef STACK_LOC_23 +#undef STACK_LOC_24 +#undef STACK_LOC_25 +#undef STACK_LOC_26 +#undef STACK_LOC_27 +#undef STACK_LOC_28 +#undef STACK_LOC_29 +#undef STACK_LOC_30 +#undef STACK_LOC_31 +#undef STACK_LOC_32 +#undef STACK_LOC_33 +#undef STACK_LOC_34 +#undef STACK_LOC_35 +#undef STACK_LOC_36 +#undef STACK_LOC_37 +#undef STACK_LOC_38 +#undef STACK_LOC_39 +#undef KECCAK_F1600_ROUNDS + +/* simpasm: footer-start */ +#endif /* MLD_FIPS202_AARCH64_NEED_X1_SCALAR && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..057d9e311b --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S @@ -0,0 +1,860 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. +// +// This is the SYMBOLIC (pre-SLOTHY) source. It must be optimized with SLOTHY +// (see the Makefile target keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S) +// to produce the interleaved production file. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations (clean version for optimization) + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t *state + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + cur_const .req x26 + + /* Mapping of Keccak-f1600 SIMD state to vector registers + * at the beginning and end of each round. Identical to x4: + * two independent states are packed into the 2 lanes of each + * 128-bit V register. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Mapping of Keccak-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +/* Vector states are state0 (at input_addr) and state1 (at input_addr+0xc8). + * Identical layout to x4. */ +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +/* The single scalar state is state2, at offset (2)*25*8 from input_addr. + * x3 only ever uses idx=0 (there is no second scalar pass). */ +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// Scalar round macros are IDENTICAL to the x4 kernel; they operate on the +// single scalar state held in GPRs. +.macro scalar_round_initial + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #1 + bic x9, x0, x16, ror #9 + str x24, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 +.endm + +// Neon round: uses the single-instruction Armv8.4-A SHA3 forms (eor3, rax1, +// xar, bcax; the "_m0" macros) throughout. Requires __ARM_FEATURE_SHA3 +// (guarded above). +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m0 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m0 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m0 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m0 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m0 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m0 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m0 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m0 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m0 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m0 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m0 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m0 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m0 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m0 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m0 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m0 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m0 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m0 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m0 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m0 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m0 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m0 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m0 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m0 vAko, vAko_, vAka_, vAku_ + bcax_m0 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m0 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m0 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m0 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m0 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m0 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m0 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m0 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #62 + eor x30, x30, x26, ror #57 + ror x26, x26, #58 + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #56 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, LSL #3] + ldr x9, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + add x25, x25, #1 + str x25, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input (state0, state1) + load_input_scalar 0 // Scalar input (state2) + + // Single pass, 1:1 interleave. + // Initial iteration: scalar_round_initial does round 0 of the scalar state + // and sets the round counter to 1; vector_round does round 0 of the two + // vector states. Then the loop runs rounds 1..23 (23 iterations), one + // scalar + one vector round each, for a total of 24 rounds on all three + // states. +mld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + final_scalar_rotate + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..fe6e6a0b04 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S @@ -0,0 +1,1031 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for four sequential states (clean version for optimization) + Signature: void mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t *state + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + outer .req x30 + cur_const .req x26 + + /* Mapping of Kecck-f1600 SIMD state to vector registers + * at the beginning and end of each round. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Unused temporary */ + vtmp .req v31 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m1 d, s0, s1, s2 + eor \d\().16b, \s0\().16b, \s1\().16b + eor \d\().16b, \d\().16b, \s2\().16b +.endm + +.macro rax1_m1 d, s0, s1 + add vtmp.2d, \s1\().2d, \s1\().2d + sri vtmp.2d, \s1\().2d, #63 + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro xar_m1 d, s0, s1, imm + eor vtmp.16b, \s0\().16b, \s1\().16b + shl \d\().2d, vtmp.2d, #(64-\imm) + sri \d\().2d, vtmp.2d, #(\imm) +.endm + +.macro bcax_m1 d, s0, s1, s2 + bic vtmp.16b, \s1\().16b, \s2\().16b + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) +#define STACK_OFFSET_OUTER (4*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// NOTE: This is copied from the de-symbolized version of +// keccak_f1600_x1_scalar_symbolic.S. When changing the scalar code, +// SLOTHY needs to be re-run to re-allocate registers and stack spills, +// and the resulting code copied here. +// +// TODO: Automate this +.macro scalar_round_initial + eor x30, x24, x25 // *..................................................... + eor x27, x9, x10 // *..................................................... + eor x0, x30, x21 // .*.................................................... + eor x26, x27, x6 // .*.................................................... + eor x27, x26, x7 // ..*................................................... + eor x29, x0, x22 // ..*................................................... + eor x26, x29, x23 // ...*.................................................. + eor x29, x4, x5 // ...*.................................................. + eor x30, x29, x1 // ....*................................................. + eor x0, x27, x8 // ....*................................................. + eor x29, x30, x2 // .....*................................................ + eor x30, x19, x20 // .....*................................................ + eor x30, x30, x16 // ......*............................................... + eor x27, x26, x0, ror #63 // ......*............................................... + eor x4, x4, x27 // .......*.............................................. + eor x30, x30, x17 // .......*.............................................. + eor x30, x30, x28 // ........*............................................. + eor x29, x29, x3 // ........*............................................. + eor x0, x0, x30, ror #63 // .........*............................................ + eor x30, x30, x29, ror #63 // .........*............................................ + eor x22, x22, x30 // ..........*........................................... + eor x23, x23, x30 // ..........*........................................... + str x23, [sp, #STACK_LOC_0] // ...........*.......................................... // @slothy:writes=stack_0 + eor x23, x14, x15 // ...........*.......................................... + eor x14, x14, x0 // ............*......................................... + eor x23, x23, x11 // ............*......................................... + eor x15, x15, x0 // .............*........................................ + eor x1, x1, x27 // .............*........................................ + eor x23, x23, x12 // ..............*....................................... + eor x23, x23, x13 // ...............*...................................... + eor x11, x11, x0 // ...............*...................................... + eor x29, x29, x23, ror #63 // ................*..................................... + eor x23, x23, x26, ror #63 // ................*..................................... + eor x26, x13, x0 // .................*.................................... + eor x13, x28, x23 // .................*.................................... + eor x28, x24, x30 // ..................*................................... + eor x24, x16, x23 // ..................*................................... + eor x16, x21, x30 // ...................*.................................. + eor x21, x25, x30 // ...................*.................................. + eor x30, x19, x23 // ....................*................................. + eor x19, x20, x23 // ....................*................................. + eor x20, x17, x23 // .....................*................................ + eor x17, x12, x0 // .....................*................................ + eor x0, x2, x27 // ......................*............................... + eor x2, x6, x29 // ......................*............................... + eor x6, x8, x29 // .......................*.............................. + bic x8, x28, x13, ror #47 // .......................*.............................. + eor x12, x3, x27 // ........................*............................. + bic x3, x13, x17, ror #19 // ........................*............................. + eor x5, x5, x27 // .........................*............................ + ldr x27, [sp, #STACK_LOC_0] // .........................*............................ // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 // ..........................*........................... + eor x9, x9, x29 // ..........................*........................... + eor x23, x25, x5, ror #52 // ...........................*.......................... + eor x3, x3, x2, ror #24 // ...........................*.......................... + eor x8, x8, x17, ror #2 // ............................*......................... + eor x17, x10, x29 // ............................*......................... + bic x25, x12, x22, ror #47 // .............................*........................ + eor x29, x7, x29 // .............................*........................ + bic x10, x4, x27, ror #2 // ..............................*....................... + bic x7, x5, x28, ror #10 // ..............................*....................... + eor x10, x10, x20, ror #50 // ...............................*...................... + eor x13, x7, x13, ror #57 // ...............................*...................... + bic x7, x2, x5, ror #47 // ................................*..................... + eor x2, x25, x24, ror #39 // ................................*..................... + bic x25, x20, x11, ror #57 // .................................*.................... + bic x5, x17, x4, ror #25 // .................................*.................... + eor x25, x25, x17, ror #53 // ..................................*................... + bic x17, x11, x17, ror #60 // ..................................*................... + eor x28, x7, x28, ror #57 // ...................................*.................. + bic x7, x9, x12, ror #42 // ...................................*.................. + eor x7, x7, x22, ror #25 // ....................................*................. + bic x22, x22, x24, ror #56 // ....................................*................. + bic x24, x24, x15, ror #31 // .....................................*................ + eor x22, x22, x15, ror #23 // .....................................*................ + bic x20, x27, x20, ror #48 // ......................................*............... + bic x15, x15, x9, ror #16 // ......................................*............... + eor x12, x15, x12, ror #58 // .......................................*.............. + eor x15, x5, x27, ror #27 // .......................................*.............. + eor x5, x20, x11, ror #41 // ........................................*............. + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ........................................*............. + eor x20, x17, x4, ror #21 // .........................................*............ + eor x17, x24, x9, ror #47 // .........................................*............ + mov x24, #1 // ..........................................*........... + bic x9, x0, x16, ror #9 // ..........................................*........... + str x24, [sp, #STACK_OFFSET_COUNT] // ...........................................*.......... // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 // ...........................................*.......... + bic x27, x1, x21, ror #50 // ............................................*......... + bic x4, x26, x29, ror #63 // ............................................*......... + eor x1, x1, x4, ror #21 // .............................................*........ + ldr x11, [x11] // .............................................*........ + bic x4, x21, x30, ror #57 // ..............................................*....... + eor x21, x24, x21, ror #30 // ..............................................*....... + eor x24, x9, x19, ror #44 // ...............................................*...... + bic x9, x14, x6, ror #5 // ...............................................*...... + eor x9, x9, x0, ror #43 // ................................................*..... + bic x0, x6, x0, ror #38 // ................................................*..... + eor x1, x1, x11 // .................................................*.... + eor x11, x4, x26, ror #35 // .................................................*.... + eor x4, x0, x16, ror #47 // ..................................................*... + bic x0, x16, x19, ror #35 // ..................................................*... + eor x16, x27, x30, ror #43 // ...................................................*.. + bic x27, x30, x26, ror #42 // ...................................................*.. + bic x26, x19, x14, ror #41 // ....................................................*. + eor x19, x0, x14, ror #12 // ....................................................*. + eor x14, x26, x6, ror #46 // .....................................................* + eor x6, x27, x29, ror #41 // .....................................................* + + // eor5 X, sAma, sAsa, tAba, sAga, sAka + // eor5 X, sAme, sAse, sAbe, sAge, sAke + // eor5 X, sAmi, sAsi, sAbi, sAgi, sAki + // eor5 X, sAmo, sAso, sAbo, sAgo, sAko + // eor5 X, sAmu, sAsu, sAbu, sAgu, sAku + + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, tAba, X + // eor X, sAbi, X + // eor X, sAki, X + // eor X, sAko, X + // eor X, sAmu, X + // eor X, sAso, X + // eor X, sAma, X + // eor X, sAbe, X + // eor X, sAgo, X + // eor X, sAme, X + // eor X, sAgi, X + // eor X, sAka, X + // eor X, sAbo, X + // eor X, sAmo, X + // eor X, sAmi, X + // eor X, sAke, X + // eor X, sAgu, X + // eor X, sAsi, X + // eor X, sAku, X + // eor X, sAsa, X + // eor X, sAbu, X + // eor X, sAsu, X + // eor X, sAse, X + // eor X, sAga, X + // eor X, sAge, X + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [X] + // mov X, #1 + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro vector_round + eor3_m1 C0, vAba, vAga, vAka + eor3_m1 C0, C0, vAma, vAsa + eor3_m1 C1, vAbe, vAge, vAke + eor3_m1 C1, C1, vAme, vAse + eor3_m1 C2, vAbi, vAgi, vAki + eor3_m1 C2, C2, vAmi, vAsi + eor3_m1 C3, vAbo, vAgo, vAko + eor3_m1 C3, C3, vAmo, vAso + eor3_m1 C4, vAbu, vAgu, vAku + eor3_m1 C4, C4, vAmu, vAsu + rax1_m1 E1, C0, C2 + rax1_m1 E3, C2, C4 + rax1_m1 E0, C4, C1 + rax1_m1 E2, C1, C3 + rax1_m1 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m1 vAsa_, vAbi, E2, 2 + xar_m1 vAbi_, vAki, E2, 21 + xar_m1 vAki_, vAko, E3, 39 + xar_m1 vAko_, vAmu, E4, 56 + xar_m1 vAmu_, vAso, E3, 8 + xar_m1 vAso_, vAma, E0, 23 + xar_m1 vAka_, vAbe, E1, 63 + xar_m1 vAse_, vAgo, E3, 9 + xar_m1 vAgo_, vAme, E1, 19 + xar_m1 vAke_, vAgi, E2, 58 + xar_m1 vAgi_, vAka, E0, 61 + xar_m1 vAga_, vAbo, E3, 36 + xar_m1 vAbo_, vAmo, E3, 43 + xar_m1 vAmo_, vAmi, E2, 49 + xar_m1 vAmi_, vAke, E1, 54 + xar_m1 vAge_, vAgu, E4, 44 + xar_m1 vAgu_, vAsi, E2, 3 + xar_m1 vAsi_, vAku, E4, 25 + xar_m1 vAku_, vAsa, E0, 46 + xar_m1 vAma_, vAbu, E4, 37 + xar_m1 vAbu_, vAsu, E4, 50 + xar_m1 vAsu_, vAse, E1, 62 + xar_m1 vAme_, vAga, E0, 28 + xar_m1 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m1 vAga, vAga_, vAgi_, vAge_ + bcax_m1 vAge, vAge_, vAgo_, vAgi_ + bcax_m1 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m1 vAgo, vAgo_, vAga_, vAgu_ + bcax_m1 vAgu, vAgu_, vAge_, vAga_ + bcax_m1 vAka, vAka_, vAki_, vAke_ + bcax_m1 vAke, vAke_, vAko_, vAki_ + bcax_m1 vAki, vAki_, vAku_, vAko_ + bcax_m1 vAko, vAko_, vAka_, vAku_ + bcax_m1 vAku, vAku_, vAke_, vAka_ + bcax_m1 vAma, vAma_, vAmi_, vAme_ + bcax_m1 vAme, vAme_, vAmo_, vAmi_ + bcax_m1 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m1 vAmo, vAmo_, vAma_, vAmu_ + bcax_m1 vAmu, vAmu_, vAme_, vAma_ + bcax_m1 vAsa, vAsa_, vAsi_, vAse_ + bcax_m1 vAse, vAse_, vAso_, vAsi_ + bcax_m1 vAsi, vAsi_, vAsu_, vAso_ + bcax_m1 vAso, vAso_, vAsa_, vAsu_ + bcax_m1 vAsu, vAsu_, vAse_, vAsa_ + bcax_m1 vAba, vAba_, vAbi_, vAbe_ + bcax_m1 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m1 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m1 vAbo, vAbo_, vAba_, vAbu_ + bcax_m1 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 // *........................................................ + eor x0, x0, x13, ror #48 // .*....................................................... + eor x26, x8, x9, ror #57 // .*....................................................... + eor x27, x0, x14, ror #10 // ..*...................................................... + eor x29, x16, x28, ror #63 // ..*...................................................... + eor x26, x26, x6, ror #51 // ...*..................................................... + eor x30, x23, x22, ror #50 // ...*..................................................... + eor x0, x26, x10, ror #31 // ....*.................................................... + eor x29, x29, x19, ror #37 // ....*.................................................... + eor x27, x27, x12, ror #5 // .....*................................................... + eor x30, x30, x24, ror #34 // .....*................................................... + eor x0, x0, x7, ror #27 // ......*.................................................. + eor x26, x30, x21, ror #26 // ......*.................................................. + eor x26, x26, x25, ror #15 // .......*................................................. + ror x30, x27, #62 // .......*................................................. + eor x30, x30, x26, ror #57 // ........*................................................ + ror x26, x26, #58 // ........*................................................ + eor x16, x30, x16 // .........*............................................... + eor x28, x30, x28, ror #63 // .........*............................................... + str x28, [sp, #STACK_LOC_0] // ..........*.............................................. // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 // ..........*.............................................. + eor x28, x1, x2, ror #61 // ...........*............................................. + eor x19, x30, x19, ror #37 // ...........*............................................. + eor x29, x29, x20, ror #2 // ............*............................................ + eor x28, x28, x4, ror #54 // ............*............................................ + eor x26, x26, x0, ror #55 // .............*........................................... + eor x28, x28, x3, ror #39 // .............*........................................... + eor x28, x28, x5, ror #25 // ..............*.......................................... + ror x0, x0, #56 // ..............*.......................................... + eor x0, x0, x29, ror #63 // ...............*......................................... + eor x27, x28, x27, ror #61 // ...............*......................................... + eor x13, x0, x13, ror #46 // ................*........................................ + eor x28, x29, x28, ror #63 // ................*........................................ + eor x29, x30, x20, ror #2 // .................*....................................... + eor x20, x26, x3, ror #39 // .................*....................................... + eor x11, x0, x11, ror #50 // ..................*...................................... + eor x25, x28, x25, ror #9 // ..................*...................................... + eor x3, x28, x21, ror #20 // ...................*..................................... + eor x21, x26, x1 // ...................*..................................... + eor x9, x27, x9, ror #49 // ....................*.................................... + eor x24, x28, x24, ror #28 // ....................*.................................... + eor x1, x30, x17, ror #36 // .....................*................................... + eor x14, x0, x14, ror #8 // .....................*................................... + eor x22, x28, x22, ror #44 // ......................*.................................. + eor x8, x27, x8, ror #56 // ......................*.................................. + eor x17, x27, x7, ror #19 // .......................*................................. + eor x15, x0, x15, ror #62 // .......................*................................. + bic x7, x20, x22, ror #47 // ........................*................................ + eor x4, x26, x4, ror #54 // ........................*................................ + eor x0, x0, x12, ror #3 // .........................*............................... + eor x28, x28, x23, ror #58 // .........................*............................... + eor x23, x26, x2, ror #61 // ..........................*.............................. + eor x26, x26, x5, ror #25 // ..........................*.............................. + eor x2, x7, x16, ror #39 // ...........................*............................. + bic x7, x9, x20, ror #42 // ...........................*............................. + bic x30, x15, x9, ror #16 // ............................*............................ + eor x7, x7, x22, ror #25 // ............................*............................ + eor x12, x30, x20, ror #58 // .............................*........................... + bic x20, x22, x16, ror #56 // .............................*........................... + eor x30, x27, x6, ror #43 // ..............................*.......................... + eor x22, x20, x15, ror #23 // ..............................*.......................... + bic x6, x19, x13, ror #42 // ...............................*......................... + eor x6, x6, x17, ror #41 // ................................*........................ + bic x5, x13, x17, ror #63 // ................................*........................ + eor x5, x21, x5, ror #21 // .................................*....................... + bic x17, x17, x21, ror #44 // .................................*....................... + eor x27, x27, x10, ror #23 // ..................................*...................... + bic x21, x21, x25, ror #50 // ..................................*...................... + bic x20, x27, x4, ror #25 // ...................................*..................... + bic x10, x16, x15, ror #31 // ...................................*..................... + eor x16, x21, x19, ror #43 // ....................................*.................... + eor x21, x17, x25, ror #30 // ....................................*.................... + bic x19, x25, x19, ror #57 // .....................................*................... + ldr x25, [sp, #STACK_OFFSET_COUNT] // .....................................*................... // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 // ......................................*.................. + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // ......................................*.................. + eor x15, x20, x28, ror #27 // .......................................*................. + bic x20, x4, x28, ror #2 // .......................................*................. + eor x10, x20, x1, ror #50 // ........................................*................ + bic x20, x11, x27, ror #60 // ........................................*................ + eor x20, x20, x4, ror #21 // .........................................*............... + bic x4, x28, x1, ror #48 // .........................................*............... + bic x1, x1, x11, ror #57 // ..........................................*.............. + ldr x28, [x9, x25, LSL #3] // ..........................................*.............. + ldr x9, [sp, #STACK_LOC_0] // ...........................................*............. // @slothy:reads=stack_0 + add x25, x25, #1 // ...........................................*............. + str x25, [sp, #STACK_OFFSET_COUNT] // ............................................*............ // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ............................................*............ // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 // .............................................*........... + bic x27, x30, x26, ror #47 // .............................................*........... + eor x1, x5, x28 // ..............................................*.......... + eor x5, x4, x11, ror #41 // ..............................................*.......... + eor x11, x19, x13, ror #35 // ...............................................*......... + bic x13, x26, x24, ror #10 // ...............................................*......... + eor x28, x27, x24, ror #57 // ................................................*........ + bic x27, x24, x9, ror #47 // ................................................*........ + bic x19, x23, x3, ror #9 // .................................................*....... + bic x4, x29, x14, ror #41 // .................................................*....... + eor x24, x19, x29, ror #44 // ..................................................*...... + bic x29, x3, x29, ror #35 // ..................................................*...... + eor x13, x13, x9, ror #57 // ...................................................*..... + eor x19, x29, x14, ror #12 // ...................................................*..... + bic x29, x9, x0, ror #19 // ....................................................*.... + bic x14, x14, x8, ror #5 // ....................................................*.... + eor x9, x14, x23, ror #43 // .....................................................*... + eor x14, x4, x8, ror #46 // .....................................................*... + bic x23, x8, x23, ror #38 // ......................................................*.. + eor x8, x27, x0, ror #2 // ......................................................*.. + eor x4, x23, x3, ror #47 // .......................................................*. + bic x3, x0, x30, ror #5 // .......................................................*. + eor x23, x3, x26, ror #52 // ........................................................* + eor x3, x29, x30, ror #24 // ........................................................* + + // eor X, tAba, sAga, ror #61 + // eor X, X, sAma, ror #54 + // eor X, X, sAka, ror #39 + // eor X, X, sAsa, ror #25 + + // eor X, sAke, sAme, ror #57 + // eor X, X, sAbe, ror #51 + // eor X, X, sAse, ror #31 + // eor X, X, sAge, ror #27 + + // eor X, sAsi, sAbi, ror #52 + // eor X, X, sAki, ror #48 + // eor X, X, sAmi, ror #10 + // eor X, X, sAgi, ror #5 + + // eor X, sAbo, sAko, ror #63 + // eor X, X, sAmo, ror #37 + // eor X, X, sAgo, ror #36 + // eor X, X, sAso, ror #2 + + // eor X, sAku, sAgu, ror #50 + // eor X, X, sAmu, ror #34 + // eor X, X, sAbu, ror #26 + // eor X, X, sAsu, ror #15 + + // eor X, X, X, ror #61 + // ror X, X, #62 + // eor X, X, X, ror #57 + // ror X, X, #58 + // eor X, X, X, ror #55 + // ror X, X, #56 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, X, tAba + // eor X, X, sAbi, ror #50 + // eor X, X, sAki, ror #46 + // eor X, X, sAko, ror #63 + // eor X, X, sAmu, ror #28 + // eor X, X, sAso, ror #2 + // eor X, X, sAma, ror #54 + // eor X, X, sAbe, ror #43 + // eor X, X, sAgo, ror #36 + // eor X, X, sAme, ror #49 + // eor X, X, sAgi, ror #3 + // eor X, X, sAka, ror #39 + // eor X, X, sAbo + // eor X, X, sAmo, ror #37 + // eor X, X, sAmi, ror #8 + // eor X, X, sAke, ror #56 + // eor X, X, sAgu, ror #44 + // eor X, X, sAsi, ror #62 + // eor X, X, sAku, ror #58 + // eor X, X, sAsa, ror #25 + // eor X, X, sAbu, ror #20 + // eor X, X, sAsu, ror #9 + // eor X, X, sAse, ror #23 + // eor X, X, sAga, ror #61 + // eor X, X, sAge, ror #19 + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + // ldr X, [X, X, LSL #3] + // add X, X, #1 + // cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + mov outer, #0 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input + load_input_scalar 0 // First scalar input + +mld_keccak_f1600_x4_v8a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x4_v8a_scalar_hybrid_loop + final_scalar_rotate + + // Read outer loop flag: We repeat the above twice + ldr outer, [sp, #STACK_OFFSET_OUTER] // @slothy:reads=STACK_OFFSET_OUTER + cmp outer, #1 + beq mld_keccak_f1600_x4_v8a_scalar_hybrid_done + + // Update outer loop flag + mov outer, #1 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 // Store first scalar data + load_input_scalar 1 // Load second scalar input + + b mld_keccak_f1600_x4_v8a_scalar_hybrid_initial +mld_keccak_f1600_x4_v8a_scalar_hybrid_done: + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 1 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq outer + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq vtmp + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_OFFSET_OUTER +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..293e246828 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S @@ -0,0 +1,1052 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations (clean version for optimization) + Signature: void mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t *state + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + outer .req x30 + cur_const .req x26 + + /* Mapping of Kecck-f1600 SIMD state to vector registers + * at the beginning and end of each round. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Unused temporary */ + vtmp .req v31 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro eor3_m1 d, s0, s1, s2 + eor \d\().16b, \s0\().16b, \s1\().16b + eor \d\().16b, \d\().16b, \s2\().16b +.endm + +.macro rax1_m1 d, s0, s1 + add vtmp.2d, \s1\().2d, \s1\().2d + sri vtmp.2d, \s1\().2d, #63 + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro xar_m1 d, s0, s1, imm + eor vtmp.16b, \s0\().16b, \s1\().16b + shl \d\().2d, vtmp.2d, #(64-\imm) + sri \d\().2d, vtmp.2d, #(\imm) +.endm + +.macro bcax_m1 d, s0, s1, s2 + bic vtmp.16b, \s1\().16b, \s2\().16b + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) +#define STACK_OFFSET_OUTER (4*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// NOTE: This is copied from the de-symbolized version of +// keccak_f1600_x1_scalar_symbolic.S. When changing the scalar code, +// SLOTHY needs to be re-run to re-allocate registers and stack spills, +// and the resulting code copied here. +// +// TODO: Automate this +.macro scalar_round_initial + eor x30, x24, x25 // *..................................................... + eor x27, x9, x10 // *..................................................... + eor x0, x30, x21 // .*.................................................... + eor x26, x27, x6 // .*.................................................... + eor x27, x26, x7 // ..*................................................... + eor x29, x0, x22 // ..*................................................... + eor x26, x29, x23 // ...*.................................................. + eor x29, x4, x5 // ...*.................................................. + eor x30, x29, x1 // ....*................................................. + eor x0, x27, x8 // ....*................................................. + eor x29, x30, x2 // .....*................................................ + eor x30, x19, x20 // .....*................................................ + eor x30, x30, x16 // ......*............................................... + eor x27, x26, x0, ror #63 // ......*............................................... + eor x4, x4, x27 // .......*.............................................. + eor x30, x30, x17 // .......*.............................................. + eor x30, x30, x28 // ........*............................................. + eor x29, x29, x3 // ........*............................................. + eor x0, x0, x30, ror #63 // .........*............................................ + eor x30, x30, x29, ror #63 // .........*............................................ + eor x22, x22, x30 // ..........*........................................... + eor x23, x23, x30 // ..........*........................................... + str x23, [sp, #STACK_LOC_0] // ...........*.......................................... // @slothy:writes=stack_0 + eor x23, x14, x15 // ...........*.......................................... + eor x14, x14, x0 // ............*......................................... + eor x23, x23, x11 // ............*......................................... + eor x15, x15, x0 // .............*........................................ + eor x1, x1, x27 // .............*........................................ + eor x23, x23, x12 // ..............*....................................... + eor x23, x23, x13 // ...............*...................................... + eor x11, x11, x0 // ...............*...................................... + eor x29, x29, x23, ror #63 // ................*..................................... + eor x23, x23, x26, ror #63 // ................*..................................... + eor x26, x13, x0 // .................*.................................... + eor x13, x28, x23 // .................*.................................... + eor x28, x24, x30 // ..................*................................... + eor x24, x16, x23 // ..................*................................... + eor x16, x21, x30 // ...................*.................................. + eor x21, x25, x30 // ...................*.................................. + eor x30, x19, x23 // ....................*................................. + eor x19, x20, x23 // ....................*................................. + eor x20, x17, x23 // .....................*................................ + eor x17, x12, x0 // .....................*................................ + eor x0, x2, x27 // ......................*............................... + eor x2, x6, x29 // ......................*............................... + eor x6, x8, x29 // .......................*.............................. + bic x8, x28, x13, ror #47 // .......................*.............................. + eor x12, x3, x27 // ........................*............................. + bic x3, x13, x17, ror #19 // ........................*............................. + eor x5, x5, x27 // .........................*............................ + ldr x27, [sp, #STACK_LOC_0] // .........................*............................ // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 // ..........................*........................... + eor x9, x9, x29 // ..........................*........................... + eor x23, x25, x5, ror #52 // ...........................*.......................... + eor x3, x3, x2, ror #24 // ...........................*.......................... + eor x8, x8, x17, ror #2 // ............................*......................... + eor x17, x10, x29 // ............................*......................... + bic x25, x12, x22, ror #47 // .............................*........................ + eor x29, x7, x29 // .............................*........................ + bic x10, x4, x27, ror #2 // ..............................*....................... + bic x7, x5, x28, ror #10 // ..............................*....................... + eor x10, x10, x20, ror #50 // ...............................*...................... + eor x13, x7, x13, ror #57 // ...............................*...................... + bic x7, x2, x5, ror #47 // ................................*..................... + eor x2, x25, x24, ror #39 // ................................*..................... + bic x25, x20, x11, ror #57 // .................................*.................... + bic x5, x17, x4, ror #25 // .................................*.................... + eor x25, x25, x17, ror #53 // ..................................*................... + bic x17, x11, x17, ror #60 // ..................................*................... + eor x28, x7, x28, ror #57 // ...................................*.................. + bic x7, x9, x12, ror #42 // ...................................*.................. + eor x7, x7, x22, ror #25 // ....................................*................. + bic x22, x22, x24, ror #56 // ....................................*................. + bic x24, x24, x15, ror #31 // .....................................*................ + eor x22, x22, x15, ror #23 // .....................................*................ + bic x20, x27, x20, ror #48 // ......................................*............... + bic x15, x15, x9, ror #16 // ......................................*............... + eor x12, x15, x12, ror #58 // .......................................*.............. + eor x15, x5, x27, ror #27 // .......................................*.............. + eor x5, x20, x11, ror #41 // ........................................*............. + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ........................................*............. + eor x20, x17, x4, ror #21 // .........................................*............ + eor x17, x24, x9, ror #47 // .........................................*............ + mov x24, #1 // ..........................................*........... + bic x9, x0, x16, ror #9 // ..........................................*........... + str x24, [sp, #STACK_OFFSET_COUNT] // ...........................................*.......... // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 // ...........................................*.......... + bic x27, x1, x21, ror #50 // ............................................*......... + bic x4, x26, x29, ror #63 // ............................................*......... + eor x1, x1, x4, ror #21 // .............................................*........ + ldr x11, [x11] // .............................................*........ + bic x4, x21, x30, ror #57 // ..............................................*....... + eor x21, x24, x21, ror #30 // ..............................................*....... + eor x24, x9, x19, ror #44 // ...............................................*...... + bic x9, x14, x6, ror #5 // ...............................................*...... + eor x9, x9, x0, ror #43 // ................................................*..... + bic x0, x6, x0, ror #38 // ................................................*..... + eor x1, x1, x11 // .................................................*.... + eor x11, x4, x26, ror #35 // .................................................*.... + eor x4, x0, x16, ror #47 // ..................................................*... + bic x0, x16, x19, ror #35 // ..................................................*... + eor x16, x27, x30, ror #43 // ...................................................*.. + bic x27, x30, x26, ror #42 // ...................................................*.. + bic x26, x19, x14, ror #41 // ....................................................*. + eor x19, x0, x14, ror #12 // ....................................................*. + eor x14, x26, x6, ror #46 // .....................................................* + eor x6, x27, x29, ror #41 // .....................................................* + + // eor5 X, sAma, sAsa, tAba, sAga, sAka + // eor5 X, sAme, sAse, sAbe, sAge, sAke + // eor5 X, sAmi, sAsi, sAbi, sAgi, sAki + // eor5 X, sAmo, sAso, sAbo, sAgo, sAko + // eor5 X, sAmu, sAsu, sAbu, sAgu, sAku + + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, tAba, X + // eor X, sAbi, X + // eor X, sAki, X + // eor X, sAko, X + // eor X, sAmu, X + // eor X, sAso, X + // eor X, sAma, X + // eor X, sAbe, X + // eor X, sAgo, X + // eor X, sAme, X + // eor X, sAgi, X + // eor X, sAka, X + // eor X, sAbo, X + // eor X, sAmo, X + // eor X, sAmi, X + // eor X, sAke, X + // eor X, sAgu, X + // eor X, sAsi, X + // eor X, sAku, X + // eor X, sAsa, X + // eor X, sAbu, X + // eor X, sAsu, X + // eor X, sAse, X + // eor X, sAga, X + // eor X, sAge, X + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [X] + // mov X, #1 + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m1 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m1 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m1 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m1 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m1 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m1 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m1 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m1 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m1 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m1 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m1 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m1 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m1 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m1 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m1 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m1 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m1 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m1 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m1 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m1 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m1 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m1 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m1 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m1 vAko, vAko_, vAka_, vAku_ + bcax_m1 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m1 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m1 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m1 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m1 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m1 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m1 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m1 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 // *........................................................ + eor x0, x0, x13, ror #48 // .*....................................................... + eor x26, x8, x9, ror #57 // .*....................................................... + eor x27, x0, x14, ror #10 // ..*...................................................... + eor x29, x16, x28, ror #63 // ..*...................................................... + eor x26, x26, x6, ror #51 // ...*..................................................... + eor x30, x23, x22, ror #50 // ...*..................................................... + eor x0, x26, x10, ror #31 // ....*.................................................... + eor x29, x29, x19, ror #37 // ....*.................................................... + eor x27, x27, x12, ror #5 // .....*................................................... + eor x30, x30, x24, ror #34 // .....*................................................... + eor x0, x0, x7, ror #27 // ......*.................................................. + eor x26, x30, x21, ror #26 // ......*.................................................. + eor x26, x26, x25, ror #15 // .......*................................................. + ror x30, x27, #62 // .......*................................................. + eor x30, x30, x26, ror #57 // ........*................................................ + ror x26, x26, #58 // ........*................................................ + eor x16, x30, x16 // .........*............................................... + eor x28, x30, x28, ror #63 // .........*............................................... + str x28, [sp, #STACK_LOC_0] // ..........*.............................................. // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 // ..........*.............................................. + eor x28, x1, x2, ror #61 // ...........*............................................. + eor x19, x30, x19, ror #37 // ...........*............................................. + eor x29, x29, x20, ror #2 // ............*............................................ + eor x28, x28, x4, ror #54 // ............*............................................ + eor x26, x26, x0, ror #55 // .............*........................................... + eor x28, x28, x3, ror #39 // .............*........................................... + eor x28, x28, x5, ror #25 // ..............*.......................................... + ror x0, x0, #56 // ..............*.......................................... + eor x0, x0, x29, ror #63 // ...............*......................................... + eor x27, x28, x27, ror #61 // ...............*......................................... + eor x13, x0, x13, ror #46 // ................*........................................ + eor x28, x29, x28, ror #63 // ................*........................................ + eor x29, x30, x20, ror #2 // .................*....................................... + eor x20, x26, x3, ror #39 // .................*....................................... + eor x11, x0, x11, ror #50 // ..................*...................................... + eor x25, x28, x25, ror #9 // ..................*...................................... + eor x3, x28, x21, ror #20 // ...................*..................................... + eor x21, x26, x1 // ...................*..................................... + eor x9, x27, x9, ror #49 // ....................*.................................... + eor x24, x28, x24, ror #28 // ....................*.................................... + eor x1, x30, x17, ror #36 // .....................*................................... + eor x14, x0, x14, ror #8 // .....................*................................... + eor x22, x28, x22, ror #44 // ......................*.................................. + eor x8, x27, x8, ror #56 // ......................*.................................. + eor x17, x27, x7, ror #19 // .......................*................................. + eor x15, x0, x15, ror #62 // .......................*................................. + bic x7, x20, x22, ror #47 // ........................*................................ + eor x4, x26, x4, ror #54 // ........................*................................ + eor x0, x0, x12, ror #3 // .........................*............................... + eor x28, x28, x23, ror #58 // .........................*............................... + eor x23, x26, x2, ror #61 // ..........................*.............................. + eor x26, x26, x5, ror #25 // ..........................*.............................. + eor x2, x7, x16, ror #39 // ...........................*............................. + bic x7, x9, x20, ror #42 // ...........................*............................. + bic x30, x15, x9, ror #16 // ............................*............................ + eor x7, x7, x22, ror #25 // ............................*............................ + eor x12, x30, x20, ror #58 // .............................*........................... + bic x20, x22, x16, ror #56 // .............................*........................... + eor x30, x27, x6, ror #43 // ..............................*.......................... + eor x22, x20, x15, ror #23 // ..............................*.......................... + bic x6, x19, x13, ror #42 // ...............................*......................... + eor x6, x6, x17, ror #41 // ................................*........................ + bic x5, x13, x17, ror #63 // ................................*........................ + eor x5, x21, x5, ror #21 // .................................*....................... + bic x17, x17, x21, ror #44 // .................................*....................... + eor x27, x27, x10, ror #23 // ..................................*...................... + bic x21, x21, x25, ror #50 // ..................................*...................... + bic x20, x27, x4, ror #25 // ...................................*..................... + bic x10, x16, x15, ror #31 // ...................................*..................... + eor x16, x21, x19, ror #43 // ....................................*.................... + eor x21, x17, x25, ror #30 // ....................................*.................... + bic x19, x25, x19, ror #57 // .....................................*................... + ldr x25, [sp, #STACK_OFFSET_COUNT] // .....................................*................... // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 // ......................................*.................. + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // ......................................*.................. + eor x15, x20, x28, ror #27 // .......................................*................. + bic x20, x4, x28, ror #2 // .......................................*................. + eor x10, x20, x1, ror #50 // ........................................*................ + bic x20, x11, x27, ror #60 // ........................................*................ + eor x20, x20, x4, ror #21 // .........................................*............... + bic x4, x28, x1, ror #48 // .........................................*............... + bic x1, x1, x11, ror #57 // ..........................................*.............. + ldr x28, [x9, x25, LSL #3] // ..........................................*.............. + ldr x9, [sp, #STACK_LOC_0] // ...........................................*............. // @slothy:reads=stack_0 + add x25, x25, #1 // ...........................................*............. + str x25, [sp, #STACK_OFFSET_COUNT] // ............................................*............ // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ............................................*............ // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 // .............................................*........... + bic x27, x30, x26, ror #47 // .............................................*........... + eor x1, x5, x28 // ..............................................*.......... + eor x5, x4, x11, ror #41 // ..............................................*.......... + eor x11, x19, x13, ror #35 // ...............................................*......... + bic x13, x26, x24, ror #10 // ...............................................*......... + eor x28, x27, x24, ror #57 // ................................................*........ + bic x27, x24, x9, ror #47 // ................................................*........ + bic x19, x23, x3, ror #9 // .................................................*....... + bic x4, x29, x14, ror #41 // .................................................*....... + eor x24, x19, x29, ror #44 // ..................................................*...... + bic x29, x3, x29, ror #35 // ..................................................*...... + eor x13, x13, x9, ror #57 // ...................................................*..... + eor x19, x29, x14, ror #12 // ...................................................*..... + bic x29, x9, x0, ror #19 // ....................................................*.... + bic x14, x14, x8, ror #5 // ....................................................*.... + eor x9, x14, x23, ror #43 // .....................................................*... + eor x14, x4, x8, ror #46 // .....................................................*... + bic x23, x8, x23, ror #38 // ......................................................*.. + eor x8, x27, x0, ror #2 // ......................................................*.. + eor x4, x23, x3, ror #47 // .......................................................*. + bic x3, x0, x30, ror #5 // .......................................................*. + eor x23, x3, x26, ror #52 // ........................................................* + eor x3, x29, x30, ror #24 // ........................................................* + + // eor X, tAba, sAga, ror #61 + // eor X, X, sAma, ror #54 + // eor X, X, sAka, ror #39 + // eor X, X, sAsa, ror #25 + + // eor X, sAke, sAme, ror #57 + // eor X, X, sAbe, ror #51 + // eor X, X, sAse, ror #31 + // eor X, X, sAge, ror #27 + + // eor X, sAsi, sAbi, ror #52 + // eor X, X, sAki, ror #48 + // eor X, X, sAmi, ror #10 + // eor X, X, sAgi, ror #5 + + // eor X, sAbo, sAko, ror #63 + // eor X, X, sAmo, ror #37 + // eor X, X, sAgo, ror #36 + // eor X, X, sAso, ror #2 + + // eor X, sAku, sAgu, ror #50 + // eor X, X, sAmu, ror #34 + // eor X, X, sAbu, ror #26 + // eor X, X, sAsu, ror #15 + + // eor X, X, X, ror #61 + // ror X, X, #62 + // eor X, X, X, ror #57 + // ror X, X, #58 + // eor X, X, X, ror #55 + // ror X, X, #56 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, X, tAba + // eor X, X, sAbi, ror #50 + // eor X, X, sAki, ror #46 + // eor X, X, sAko, ror #63 + // eor X, X, sAmu, ror #28 + // eor X, X, sAso, ror #2 + // eor X, X, sAma, ror #54 + // eor X, X, sAbe, ror #43 + // eor X, X, sAgo, ror #36 + // eor X, X, sAme, ror #49 + // eor X, X, sAgi, ror #3 + // eor X, X, sAka, ror #39 + // eor X, X, sAbo + // eor X, X, sAmo, ror #37 + // eor X, X, sAmi, ror #8 + // eor X, X, sAke, ror #56 + // eor X, X, sAgu, ror #44 + // eor X, X, sAsi, ror #62 + // eor X, X, sAku, ror #58 + // eor X, X, sAsa, ror #25 + // eor X, X, sAbu, ror #20 + // eor X, X, sAsu, ror #9 + // eor X, X, sAse, ror #23 + // eor X, X, sAga, ror #61 + // eor X, X, sAge, ror #19 + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + // ldr X, [X, X, LSL #3] + // add X, X, #1 + // cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + mov outer, #0 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input + load_input_scalar 0 // First scalar input + +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + final_scalar_rotate + + // Read outer loop flag: We repeat the above twice + ldr outer, [sp, #STACK_OFFSET_OUTER] // @slothy:reads=STACK_OFFSET_OUTER + cmp outer, #1 + beq mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done + + // Update outer loop flag + mov outer, #1 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 // Store first scalar data + load_input_scalar 1 // Load second scalar input + + b mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done: + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 1 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq outer + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq vtmp + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_OFFSET_OUTER +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/examples/basic_deterministic/mldsa_native/mldsa_native_config.h b/examples/basic_deterministic/mldsa_native/mldsa_native_config.h index 99f4062e34..ebf84c8a8f 100644 --- a/examples/basic_deterministic/mldsa_native/mldsa_native_config.h +++ b/examples/basic_deterministic/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/basic_lowram/mldsa_native/mldsa_native_config.h b/examples/basic_lowram/mldsa_native/mldsa_native_config.h index fa41d68f15..396fa3d538 100644 --- a/examples/basic_lowram/mldsa_native/mldsa_native_config.h +++ b/examples/basic_lowram/mldsa_native/mldsa_native_config.h @@ -413,6 +413,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h b/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h index 5da4634b8e..cec3342d88 100644 --- a/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h +++ b/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h b/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h index efb55b8c2f..9da43a32cc 100644 --- a/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h +++ b/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h @@ -412,6 +412,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/context_parameter/mldsa_native/mldsa_native_config.h b/examples/context_parameter/mldsa_native/mldsa_native_config.h index c5e715109e..034a0e657d 100644 --- a/examples/context_parameter/mldsa_native/mldsa_native_config.h +++ b/examples/context_parameter/mldsa_native/mldsa_native_config.h @@ -417,6 +417,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/custom_backend/mldsa_native/mldsa_native_config.h b/examples/custom_backend/mldsa_native/mldsa_native_config.h index e8527151b2..3e628be55b 100644 --- a/examples/custom_backend/mldsa_native/mldsa_native_config.h +++ b/examples/custom_backend/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build/mldsa_native/mldsa_native_config.h b/examples/monolithic_build/mldsa_native/mldsa_native_config.h index a500982e38..5f8b2031d7 100644 --- a/examples/monolithic_build/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h index 7d4094b8df..b43576b386 100644 --- a/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h index 3ee4024d6c..34eb274b5f 100644 --- a/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h index a98f9e1f3d..44858e959c 100644 --- a/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h @@ -408,6 +408,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/multilevel_build/mldsa_native/mldsa_native_config.h b/examples/multilevel_build/mldsa_native/mldsa_native_config.h index 3231525631..09609ab84e 100644 --- a/examples/multilevel_build/mldsa_native/mldsa_native_config.h +++ b/examples/multilevel_build/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h b/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h index e1a04fb214..7e98ace116 100644 --- a/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h +++ b/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h @@ -408,6 +408,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/restartable_sign/mldsa_native/mldsa_native_config.h b/examples/restartable_sign/mldsa_native/mldsa_native_config.h index 39a7b40733..0d1e912851 100644 --- a/examples/restartable_sign/mldsa_native/mldsa_native_config.h +++ b/examples/restartable_sign/mldsa_native/mldsa_native_config.h @@ -412,6 +412,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/mldsa/mldsa_native.c b/mldsa/mldsa_native.c index 59a91ac2bc..0d25663fe1 100644 --- a/mldsa/mldsa_native.c +++ b/mldsa/mldsa_native.c @@ -70,9 +70,10 @@ #if !defined(MLD_CONFIG_FIPS202_CUSTOM_HEADER) #include "src/fips202/fips202.c" +#include "src/fips202/fips202x3.c" #include "src/fips202/fips202x4.c" #include "src/fips202/keccakf1600.c" -#endif +#endif /* !MLD_CONFIG_FIPS202_CUSTOM_HEADER */ #if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) #if defined(MLD_SYS_AARCH64) @@ -317,8 +318,10 @@ #undef mld_poly_challenge #undef mld_poly_decompose #undef mld_poly_uniform_eta +#undef mld_poly_uniform_eta_3x #undef mld_poly_uniform_eta_4x #undef mld_poly_uniform_gamma1 +#undef mld_poly_uniform_gamma1_3x #undef mld_poly_uniform_gamma1_4x #undef mld_poly_use_hint #undef mld_polyeta_pack @@ -486,6 +489,7 @@ #undef mld_poly_shiftl #undef mld_poly_sub #undef mld_poly_uniform +#undef mld_poly_uniform_3x #undef mld_poly_uniform_4x #undef mld_polyt0_pack #undef mld_polyt0_unpack @@ -510,6 +514,11 @@ #undef mld_xof128_init #undef mld_xof128_release #undef mld_xof128_squeezeblocks +#undef mld_xof128_x3_absorb +#undef mld_xof128_x3_ctx +#undef mld_xof128_x3_init +#undef mld_xof128_x3_release +#undef mld_xof128_x3_squeezeblocks #undef mld_xof128_x4_absorb #undef mld_xof128_x4_ctx #undef mld_xof128_x4_init @@ -520,6 +529,11 @@ #undef mld_xof256_init #undef mld_xof256_release #undef mld_xof256_squeezeblocks +#undef mld_xof256_x3_absorb +#undef mld_xof256_x3_ctx +#undef mld_xof256_x3_init +#undef mld_xof256_x3_release +#undef mld_xof256_x3_squeezeblocks #undef mld_xof256_x4_absorb #undef mld_xof256_x4_ctx #undef mld_xof256_x4_init @@ -554,6 +568,16 @@ #undef mld_shake256_init #undef mld_shake256_release #undef mld_shake256_squeeze +/* mldsa/src/fips202/fips202x3.h */ +#undef MLD_FIPS202_FIPS202X3_H +#undef mld_shake128x3_absorb_once +#undef mld_shake128x3_init +#undef mld_shake128x3_release +#undef mld_shake128x3_squeezeblocks +#undef mld_shake256x3_absorb_once +#undef mld_shake256x3_init +#undef mld_shake256x3_release +#undef mld_shake256x3_squeezeblocks /* mldsa/src/fips202/fips202x4.h */ #undef MLD_FIPS202_FIPS202X4_H #undef mld_shake128x4_absorb_once @@ -568,9 +592,13 @@ #undef MLD_FIPS202_KECCAKF1600_H #undef MLD_KECCAK_LANES #undef MLD_KECCAK_WAY +#undef MLD_KECCAK_WAY_X3 #undef mld_keccakf1600_extract_bytes #undef mld_keccakf1600_permute #undef mld_keccakf1600_xor_bytes +#undef mld_keccakf1600x3_extract_bytes +#undef mld_keccakf1600x3_permute +#undef mld_keccakf1600x3_xor_bytes #undef mld_keccakf1600x4_extract_bytes #undef mld_keccakf1600x4_permute #undef mld_keccakf1600x4_xor_bytes @@ -594,6 +622,7 @@ #undef mld_keccak_f1600_x1_scalar_aarch64_asm #undef mld_keccak_f1600_x1_v84a_aarch64_asm #undef mld_keccak_f1600_x2_v84a_aarch64_asm +#undef mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm #undef mld_keccakf1600_round_constants @@ -609,6 +638,10 @@ #undef MLD_FIPS202_AARCH64_NEED_X2_V84A #undef MLD_FIPS202_NATIVE_AARCH64_X2_V84A_H #undef MLD_USE_NATIVE_FIPS202_X4 +/* mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h */ +#undef MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID +#undef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#undef MLD_USE_NATIVE_FIPS202_X3 /* mldsa/src/fips202/native/aarch64/x4_v8a_scalar.h */ #undef MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID #undef MLD_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H diff --git a/mldsa/mldsa_native_asm.S b/mldsa/mldsa_native_asm.S index 4d6ad65878..0fd936bef4 100644 --- a/mldsa/mldsa_native_asm.S +++ b/mldsa/mldsa_native_asm.S @@ -112,6 +112,7 @@ #include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S" #endif /* MLD_SYS_AARCH64 */ @@ -355,8 +356,10 @@ #undef mld_poly_challenge #undef mld_poly_decompose #undef mld_poly_uniform_eta +#undef mld_poly_uniform_eta_3x #undef mld_poly_uniform_eta_4x #undef mld_poly_uniform_gamma1 +#undef mld_poly_uniform_gamma1_3x #undef mld_poly_uniform_gamma1_4x #undef mld_poly_use_hint #undef mld_polyeta_pack @@ -524,6 +527,7 @@ #undef mld_poly_shiftl #undef mld_poly_sub #undef mld_poly_uniform +#undef mld_poly_uniform_3x #undef mld_poly_uniform_4x #undef mld_polyt0_pack #undef mld_polyt0_unpack @@ -548,6 +552,11 @@ #undef mld_xof128_init #undef mld_xof128_release #undef mld_xof128_squeezeblocks +#undef mld_xof128_x3_absorb +#undef mld_xof128_x3_ctx +#undef mld_xof128_x3_init +#undef mld_xof128_x3_release +#undef mld_xof128_x3_squeezeblocks #undef mld_xof128_x4_absorb #undef mld_xof128_x4_ctx #undef mld_xof128_x4_init @@ -558,6 +567,11 @@ #undef mld_xof256_init #undef mld_xof256_release #undef mld_xof256_squeezeblocks +#undef mld_xof256_x3_absorb +#undef mld_xof256_x3_ctx +#undef mld_xof256_x3_init +#undef mld_xof256_x3_release +#undef mld_xof256_x3_squeezeblocks #undef mld_xof256_x4_absorb #undef mld_xof256_x4_ctx #undef mld_xof256_x4_init @@ -592,6 +606,16 @@ #undef mld_shake256_init #undef mld_shake256_release #undef mld_shake256_squeeze +/* mldsa/src/fips202/fips202x3.h */ +#undef MLD_FIPS202_FIPS202X3_H +#undef mld_shake128x3_absorb_once +#undef mld_shake128x3_init +#undef mld_shake128x3_release +#undef mld_shake128x3_squeezeblocks +#undef mld_shake256x3_absorb_once +#undef mld_shake256x3_init +#undef mld_shake256x3_release +#undef mld_shake256x3_squeezeblocks /* mldsa/src/fips202/fips202x4.h */ #undef MLD_FIPS202_FIPS202X4_H #undef mld_shake128x4_absorb_once @@ -606,9 +630,13 @@ #undef MLD_FIPS202_KECCAKF1600_H #undef MLD_KECCAK_LANES #undef MLD_KECCAK_WAY +#undef MLD_KECCAK_WAY_X3 #undef mld_keccakf1600_extract_bytes #undef mld_keccakf1600_permute #undef mld_keccakf1600_xor_bytes +#undef mld_keccakf1600x3_extract_bytes +#undef mld_keccakf1600x3_permute +#undef mld_keccakf1600x3_xor_bytes #undef mld_keccakf1600x4_extract_bytes #undef mld_keccakf1600x4_permute #undef mld_keccakf1600x4_xor_bytes @@ -632,6 +660,7 @@ #undef mld_keccak_f1600_x1_scalar_aarch64_asm #undef mld_keccak_f1600_x1_v84a_aarch64_asm #undef mld_keccak_f1600_x2_v84a_aarch64_asm +#undef mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm #undef mld_keccakf1600_round_constants @@ -647,6 +676,10 @@ #undef MLD_FIPS202_AARCH64_NEED_X2_V84A #undef MLD_FIPS202_NATIVE_AARCH64_X2_V84A_H #undef MLD_USE_NATIVE_FIPS202_X4 +/* mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h */ +#undef MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID +#undef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#undef MLD_USE_NATIVE_FIPS202_X3 /* mldsa/src/fips202/native/aarch64/x4_v8a_scalar.h */ #undef MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID #undef MLD_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H diff --git a/mldsa/mldsa_native_config.h b/mldsa/mldsa_native_config.h index 1baba77f66..3e5c0636e7 100644 --- a/mldsa/mldsa_native_config.h +++ b/mldsa/mldsa_native_config.h @@ -395,6 +395,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/mldsa/src/fips202/fips202x3.c b/mldsa/src/fips202/fips202x3.c new file mode 100644 index 0000000000..181d889bce --- /dev/null +++ b/mldsa/src/fips202/fips202x3.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + +/* + * 3-way batched SHAKE128/SHAKE256 (MLD_CONFIG_USE_X3), mirroring fips202x4.c + * but 3-wide. See fips202x3.h. + */ + +#include "../common.h" +#if !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && defined(MLD_CONFIG_USE_X3) + +#include "../ct.h" +#include "fips202.h" +#include "fips202x3.h" +#include "keccakf1600.h" + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ + defined(MLD_UNIT_TEST) +static void mld_keccak_absorb_once_x3(uint64_t *s, unsigned r, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, size_t inlen, + uint8_t p) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLD_KECCAK_LANES) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3))) +{ + while (inlen >= r) + __loop__( + assigns(inlen, in0, in1, in2, memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + invariant(inlen <= loop_entry(inlen)) + invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen)) + invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen)) + invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen)) + decreases(inlen)) + { + mld_keccakf1600x3_xor_bytes(s, in0, in1, in2, 0, r); + mld_keccakf1600x3_permute(s); + + in0 += r; + in1 += r; + in2 += r; + inlen -= r; + } + + /* Safety: At this point, inlen < r, so the truncations to unsigned are safe + * below. */ + if (inlen > 0) + { + mld_keccakf1600x3_xor_bytes(s, in0, in1, in2, 0, (unsigned)inlen); + } + + if (inlen == r - 1) + { + p |= 128; + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, (unsigned)inlen, 1); + } + else + { + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, (unsigned)inlen, 1); + p = 128; + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, r - 1, 1); + } +} + +static void mld_keccak_squeezeblocks_x3(uint8_t *out0, uint8_t *out1, + uint8_t *out2, size_t nblocks, + uint64_t *s, unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLD_KECCAK_LANES) + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(out0, nblocks * r)) + requires(memory_no_alias(out1, nblocks * r)) + requires(memory_no_alias(out2, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + assigns(memory_slice(out0, nblocks * r)) + assigns(memory_slice(out1, nblocks * r)) + assigns(memory_slice(out2, nblocks * r))) +{ + size_t current_offset = 0; + while (nblocks > 0) + __loop__( + assigns(nblocks, current_offset, + memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3), + memory_slice(out0, nblocks * r), + memory_slice(out1, nblocks * r), + memory_slice(out2, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks)) + invariant(current_offset == (loop_entry(nblocks) - nblocks) * r) + decreases(nblocks)) + { + mld_keccakf1600x3_permute(s); + mld_keccakf1600x3_extract_bytes(s, &out0[current_offset], + &out1[current_offset], + &out2[current_offset], 0, r); + current_offset += r; + nblocks--; + } +} +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ + +#if !defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +MLD_INTERNAL_API +void mld_shake128x3_absorb_once(mld_shake128x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +{ + mld_memset(state, 0, sizeof(mld_shake128x3ctx)); + mld_keccak_absorb_once_x3(state->ctx, SHAKE128_RATE, in0, in1, in2, inlen, + 0x1F); +} + +MLD_INTERNAL_API +void mld_shake128x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake128x3ctx *state) +{ + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, state->ctx, + SHAKE128_RATE); +} + +MLD_INTERNAL_API +void mld_shake128x3_init(mld_shake128x3ctx *state) { (void)state; } +MLD_INTERNAL_API +void mld_shake128x3_release(mld_shake128x3ctx *state) +{ + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(state, sizeof(mld_shake128x3ctx)); +} +#endif /* !MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || \ + (!defined(MLD_CONFIG_NO_SIGN_API) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST))) +MLD_INTERNAL_API +void mld_shake256x3_absorb_once(mld_shake256x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +{ + mld_memset(state, 0, sizeof(mld_shake256x3ctx)); + mld_keccak_absorb_once_x3(state->ctx, SHAKE256_RATE, in0, in1, in2, inlen, + 0x1F); +} + +MLD_INTERNAL_API +void mld_shake256x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake256x3ctx *state) +{ + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, state->ctx, + SHAKE256_RATE); +} + +MLD_INTERNAL_API +void mld_shake256x3_init(mld_shake256x3ctx *state) { (void)state; } +MLD_INTERNAL_API +void mld_shake256x3_release(mld_shake256x3ctx *state) +{ + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(state, sizeof(mld_shake256x3ctx)); +} +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ + +#endif /* !MLD_CONFIG_MULTILEVEL_NO_SHARED && !MLD_CONFIG_SERIAL_FIPS202_ONLY \ + && MLD_CONFIG_USE_X3 */ diff --git a/mldsa/src/fips202/fips202x3.h b/mldsa/src/fips202/fips202x3.h new file mode 100644 index 0000000000..523b85ad27 --- /dev/null +++ b/mldsa/src/fips202/fips202x3.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_FIPS202_FIPS202X3_H +#define MLD_FIPS202_FIPS202X3_H + +#include "../common.h" + +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && defined(MLD_CONFIG_USE_X3) + +#include + +#include "../cbmc.h" +#include "fips202.h" +#include "keccakf1600.h" + +/* + * 3-way batched SHAKE (MLD_CONFIG_USE_X3), mirroring fips202x4 but 3-wide. + * Provides SHAKE128 (x3 ExpandA: mld_poly_uniform_3x) and SHAKE256 (x3 + * ExpandS and ExpandMask: mld_poly_uniform_eta_3x, mld_poly_uniform_gamma1_3x). + * Mirrors the SHAKE128/SHAKE256 blocks in fips202x4.h. + */ + +/** Context for the non-incremental 3-way SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; /**< 3-way Keccak state, + stored sequentially. */ +} mld_shake128x3ctx; + +#if !defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +#define mld_shake128x3_absorb_once MLD_NAMESPACE(shake128x3_absorb_once) +MLD_INTERNAL_API +void mld_shake128x3_absorb_once(mld_shake128x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mld_shake128x3ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(state, sizeof(mld_shake128x3ctx))) +); + +#define mld_shake128x3_squeezeblocks MLD_NAMESPACE(shake128x3_squeezeblocks) +MLD_INTERNAL_API +void mld_shake128x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake128x3ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mld_shake128x3ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE128_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE128_RATE), + memory_slice(out1, nblocks * SHAKE128_RATE), + memory_slice(out2, nblocks * SHAKE128_RATE), + memory_slice(state, sizeof(mld_shake128x3ctx))) +); + +#define mld_shake128x3_init MLD_NAMESPACE(shake128x3_init) +MLD_INTERNAL_API +void mld_shake128x3_init(mld_shake128x3ctx *state); + +#define mld_shake128x3_release MLD_NAMESPACE(shake128x3_release) +MLD_INTERNAL_API +void mld_shake128x3_release(mld_shake128x3ctx *state); +#endif /* !MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +/** Context for the non-incremental 3-way SHAKE256 API. */ +typedef struct +{ + uint64_t ctx[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; /**< 3-way Keccak state, + stored sequentially. */ +} mld_shake256x3ctx; + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || \ + (!defined(MLD_CONFIG_NO_SIGN_API) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST))) +#define mld_shake256x3_absorb_once MLD_NAMESPACE(shake256x3_absorb_once) +MLD_INTERNAL_API +void mld_shake256x3_absorb_once(mld_shake256x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mld_shake256x3ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(state, sizeof(mld_shake256x3ctx))) +); + +#define mld_shake256x3_squeezeblocks MLD_NAMESPACE(shake256x3_squeezeblocks) +MLD_INTERNAL_API +void mld_shake256x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake256x3ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mld_shake256x3ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE256_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE256_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE256_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE256_RATE), + memory_slice(out1, nblocks * SHAKE256_RATE), + memory_slice(out2, nblocks * SHAKE256_RATE), + memory_slice(state, sizeof(mld_shake256x3ctx))) +); + +#define mld_shake256x3_init MLD_NAMESPACE(shake256x3_init) +MLD_INTERNAL_API +void mld_shake256x3_init(mld_shake256x3ctx *state); + +#define mld_shake256x3_release MLD_NAMESPACE(shake256x3_release) +MLD_INTERNAL_API +void mld_shake256x3_release(mld_shake256x3ctx *state); +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ + +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ +#endif /* !MLD_FIPS202_FIPS202X3_H */ diff --git a/mldsa/src/fips202/fips202x4.c b/mldsa/src/fips202/fips202x4.c index 23da9ac800..c41a451730 100644 --- a/mldsa/src/fips202/fips202x4.c +++ b/mldsa/src/fips202/fips202x4.c @@ -15,7 +15,7 @@ #include "../common.h" #if !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ - !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) + !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && !defined(MLD_CONFIG_USE_X3) #include "../ct.h" #include "fips202.h" @@ -184,4 +184,4 @@ void mld_shake256x4_release(mld_shake256x4ctx *state) (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ #endif /* !MLD_CONFIG_MULTILEVEL_NO_SHARED && !MLD_CONFIG_SERIAL_FIPS202_ONLY \ - */ + && !MLD_CONFIG_USE_X3 */ diff --git a/mldsa/src/fips202/fips202x4.h b/mldsa/src/fips202/fips202x4.h index 2f5db18e6a..4814508770 100644 --- a/mldsa/src/fips202/fips202x4.h +++ b/mldsa/src/fips202/fips202x4.h @@ -8,7 +8,7 @@ #include "../common.h" -#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && !defined(MLD_CONFIG_USE_X3) #include @@ -121,5 +121,5 @@ void mld_shake256x4_release(mld_shake256x4ctx *state); #endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ #endif /* !MLD_FIPS202_FIPS202X4_H */ diff --git a/mldsa/src/fips202/keccakf1600.c b/mldsa/src/fips202/keccakf1600.c index 18d2e6e04b..2cfeec8c9d 100644 --- a/mldsa/src/fips202/keccakf1600.c +++ b/mldsa/src/fips202/keccakf1600.c @@ -85,6 +85,7 @@ void mld_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, #if (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ defined(MLD_UNIT_TEST)) && \ !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) || defined(MLD_UNIT_TEST) static void mld_keccakf1600x4_extract_bytes_c(uint64_t *state, unsigned char *data0, unsigned char *data1, @@ -197,6 +198,96 @@ void mld_keccakf1600x4_permute(uint64_t *state) mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 2); mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 3); } +#endif /* !MLD_CONFIG_USE_X3 || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_USE_X3) +/* 3-way (x3) batched Keccak primitives (see keccakf1600.h). + * xor_bytes / extract_bytes have no native x3 variant (like x4); they loop the + * serial primitive over the 3 sub-states. permute dispatches to the native x3 + * kernel when MLD_USE_NATIVE_FIPS202_X3 is defined (MLD_CONFIG_USE_X3 on + * AArch64), else falls back to 3x serial permute. */ +static void mld_keccakf1600x3_extract_bytes_c(uint64_t *state, + unsigned char *data0, + unsigned char *data1, + unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) +) +{ + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 0, data0, offset, + length); + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 1, data1, offset, + length); + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 2, data2, offset, + length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned offset, unsigned length) +{ + mld_keccakf1600x3_extract_bytes_c(state, data0, data1, data2, offset, length); +} + +static void mld_keccakf1600x3_xor_bytes_c(uint64_t *state, + const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +) +{ + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 0, data0, offset, + length); + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 1, data1, offset, + length); + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 2, data2, offset, + length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length) +{ + mld_keccakf1600x3_xor_bytes_c(state, data0, data1, data2, offset, length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_permute(uint64_t *state) +{ +#if defined(MLD_USE_NATIVE_FIPS202_X3) + if (mld_keccak_f1600_x3_native(state) == MLD_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLD_USE_NATIVE_FIPS202_X3 */ + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 0); + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 1); + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 2); +} +#endif /* MLD_CONFIG_USE_X3 */ #endif /* (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) && !MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/fips202/keccakf1600.h b/mldsa/src/fips202/keccakf1600.h index c0875f4a6b..d9590e6359 100644 --- a/mldsa/src/fips202/keccakf1600.h +++ b/mldsa/src/fips202/keccakf1600.h @@ -12,6 +12,11 @@ #define MLD_KECCAK_LANES 25 #define MLD_KECCAK_WAY 4 +/* Batching width of the 3-way (x3) Keccak path used with MLD_CONFIG_USE_X3. + * Kept separate from MLD_KECCAK_WAY (=4), which the x4 API and its contracts + * depend on. */ +#define MLD_KECCAK_WAY_X3 3 + /* * WARNING: * The contents of this structure, including the placement @@ -46,6 +51,7 @@ __contract__( #if (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ defined(MLD_UNIT_TEST)) && \ !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) || defined(MLD_UNIT_TEST) #define mld_keccakf1600x4_extract_bytes \ MLD_NAMESPACE(keccakf1600x4_extract_bytes) MLD_INTERNAL_API @@ -96,6 +102,57 @@ __contract__( requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) ); +#endif /* !MLD_CONFIG_USE_X3 || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_USE_X3) +/* 3-way (x3) batched Keccak primitives, used with MLD_CONFIG_USE_X3. They + * mirror the x4 trio above but operate on 3 sequential states + * (MLD_KECCAK_WAY_X3), and back the 3-way SHAKE128/SHAKE256 in fips202x3. */ +#define mld_keccakf1600x3_extract_bytes \ + MLD_NAMESPACE(keccakf1600x3_extract_bytes) +MLD_INTERNAL_API +void mld_keccakf1600x3_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) +); + +#define mld_keccakf1600x3_xor_bytes MLD_NAMESPACE(keccakf1600x3_xor_bytes) +MLD_INTERNAL_API +void mld_keccakf1600x3_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +); + +#define mld_keccakf1600x3_permute MLD_NAMESPACE(keccakf1600x3_permute) +MLD_INTERNAL_API +void mld_keccakf1600x3_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) && !MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/fips202/native/aarch64/auto.h b/mldsa/src/fips202/native/aarch64/auto.h index dffe921385..eeaae6a2d4 100644 --- a/mldsa/src/fips202/native/aarch64/auto.h +++ b/mldsa/src/fips202/native/aarch64/auto.h @@ -67,9 +67,17 @@ */ #if defined(__APPLE__) || defined(MLD_SYS_AARCH64_FAST_SHA3) #include "x2_v84a.h" -#else +#elif defined(MLD_CONFIG_USE_X3) +/* + * 3-way hybrid (two states on Neon with SHA3, one on scalar registers, single + * pass), selected with MLD_CONFIG_USE_X3. Intended for cores that execute the + * SHA3 instructions on all SIMD pipes (e.g. Neoverse V3). When + * MLD_CONFIG_USE_X3 is not set, the x4 hybrid below is used. + */ +#include "x3_v84a_scalar.h" +#else /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && MLD_CONFIG_USE_X3 */ #include "x4_v8a_v84a_scalar.h" -#endif +#endif /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && !MLD_CONFIG_USE_X3 */ #else /* __ARM_FEATURE_SHA3 */ diff --git a/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h b/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h index ff6d963eff..b000f8b994 100644 --- a/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h +++ b/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -66,4 +66,18 @@ __contract__( assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) ); +/* + * 3-way hybrid Keccak-f1600: states 0 and 1 on Neon (Armv8.4-A SHA3), state 2 + * on scalar registers with lazy rotations, interleaved 1:1 in a single pass. + */ +#define mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm \ + MLD_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) +); + #endif /* !MLD_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..f2575f7bd6 --- /dev/null +++ b/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,647 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x27, x26, x7 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x29, x0, x22 + eor x26, x29, x23 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x29, x4, x5 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x29, x30, x2 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x27, x26, x0, ror #63 + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x4, x4, x27 + eor x30, x30, x17 + rax1 v25.2d, v30.2d, v28.2d + eor x30, x30, x28 + eor x29, x29, x3 + rax1 v28.2d, v28.2d, v26.2d + eor x0, x0, x30, ror #63 + rax1 v26.2d, v26.2d, v29.2d + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + rax1 v29.2d, v29.2d, v27.2d + eor x23, x23, x30 + rax1 v27.2d, v27.2d, v30.2d + str x23, [sp, #0xd0] + eor x23, x14, x15 + eor v30.16b, v0.16b, v26.16b + eor x14, x14, x0 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x23, x23, x11 + eor x15, x15, x0 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x1, x1, x27 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x23, x23, x12 + eor x23, x23, x13 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x23, x23, x26, ror #63 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x26, x13, x0 + eor x13, x28, x23 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x28, x24, x30 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x24, x16, x23 + eor x16, x21, x30 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x21, x25, x30 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x30, x19, x23 + eor x19, x20, x23 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x20, x17, x23 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x17, x12, x0 + eor x0, x2, x27 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x2, x6, x29 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x5, x5, x27 + xar v9.2d, v22.2d, v29.2d, #0x3 + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x9, x9, x29 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x8, x8, x17, ror #2 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x7, x29 + xar v21.2d, v5.2d, v26.2d, #0x1c + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x7, x2, x5, ror #47 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x5, x17, x4, ror #25 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + bcax v9.16b, v9.16b, v11.16b, v10.16b + eor x28, x7, x28, ror #57 + bcax v10.16b, v15.16b, v12.16b, v16.16b + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bcax v11.16b, v16.16b, v13.16b, v12.16b + bic x22, x22, x24, ror #56 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bcax v13.16b, v13.16b, v15.16b, v14.16b + bic x20, x27, x20, ror #48 + bcax v14.16b, v14.16b, v16.16b, v15.16b + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x11, [sp, #0x8] + bcax v17.16b, v17.16b, v19.16b, v18.16b + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + bcax v18.16b, v18.16b, v20.16b, v19.16b + mov x24, #0x1 // =1 + bcax v19.16b, v19.16b, v21.16b, v20.16b + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + bcax v20.16b, v0.16b, v22.16b, v1.16b + bic x24, x29, x1, ror #44 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + bcax v22.16b, v22.16b, v24.16b, v23.16b + eor x1, x1, x4, ror #21 + bcax v23.16b, v23.16b, v0.16b, v24.16b + ldr x11, [x11] + bic x4, x21, x30, ror #57 + bcax v24.16b, v24.16b, v1.16b, v0.16b + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bcax v0.16b, v30.16b, v2.16b, v27.16b + bic x9, x14, x6, ror #5 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x1, x1, x11 + bcax v3.16b, v3.16b, v30.16b, v4.16b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + bic x26, x19, x14, ror #41 + str x30, [sp, #0x10] + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor v0.16b, v0.16b, v28.16b + eor x6, x27, x29, ror #41 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x29, x16, x28, ror #63 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x27, x27, x12, ror #5 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x26, x30, x21, ror #26 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + rax1 v25.2d, v30.2d, v28.2d + eor x16, x30, x16 + rax1 v28.2d, v28.2d, v26.2d + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + rax1 v26.2d, v26.2d, v29.2d + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + rax1 v29.2d, v29.2d, v27.2d + eor x19, x30, x19, ror #37 + rax1 v27.2d, v27.2d, v30.2d + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x26, x26, x0, ror #55 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + xar v2.2d, v12.2d, v29.2d, #0x15 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x27, x28, x27, ror #61 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x11, x0, x11, ror #50 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x21, x26, x1 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x22, x28, x22, ror #44 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x4, x26, x4, ror #54 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x26, x2, ror #61 + xar v9.2d, v22.2d, v29.2d, #0x3 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + xar v22.2d, v14.2d, v27.2d, #0x19 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x7, x7, x22, ror #25 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + xar v24.2d, v21.2d, v25.2d, #0x3e + bic x6, x19, x13, ror #42 + xar v21.2d, v5.2d, v26.2d, #0x1c + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x5, x21, x5, ror #21 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bcax v6.16b, v11.16b, v8.16b, v7.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x10, x16, x15, ror #31 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bcax v9.16b, v9.16b, v11.16b, v10.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x17, x10, x9, ror #47 + bcax v11.16b, v16.16b, v13.16b, v12.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x20, x4, x28, ror #2 + bcax v13.16b, v13.16b, v15.16b, v14.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v14.16b, v14.16b, v16.16b, v15.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x1, x1, x11, ror #57 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v17.16b, v17.16b, v19.16b, v18.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bcax v18.16b, v18.16b, v20.16b, v19.16b + cmp x25, #0x17 + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + bcax v20.16b, v0.16b, v22.16b, v1.16b + eor x1, x5, x28 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bcax v22.16b, v22.16b, v24.16b, v23.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bcax v23.16b, v23.16b, v0.16b, v24.16b + bic x27, x24, x9, ror #47 + bcax v24.16b, v24.16b, v1.16b, v0.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x13, x13, x9, ror #57 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v3.16b, v3.16b, v30.16b, v4.16b + bic x14, x14, x8, ror #5 + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + b.le Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLD_ASM_FN_SIZE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c b/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c index eb578bcfcd..df0652c1f4 100644 --- a/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c +++ b/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c @@ -14,6 +14,7 @@ #if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \ defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \ defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) @@ -34,7 +35,8 @@ MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t #else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED */ @@ -42,6 +44,7 @@ MLD_EMPTY_CU(fips202_aarch64_round_constants) #endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h b/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h new file mode 100644 index 0000000000..fa3250a0c1 --- /dev/null +++ b/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#define MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X3 +/* Guard for assembly file */ +#define MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || + !mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm( + state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H */ diff --git a/mldsa/src/fips202/native/api.h b/mldsa/src/fips202/native/api.h index 7abac85a4e..36958d5ca6 100644 --- a/mldsa/src/fips202/native/api.h +++ b/mldsa/src/fips202/native/api.h @@ -65,6 +65,21 @@ __contract__( ensures((return_value == MLD_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4)) ); #endif /* MLD_USE_NATIVE_FIPS202_X4 */ +/* + * 3-fold batched Keccak-F1600 (used with MLD_CONFIG_USE_X3). To enable, set + * MLD_USE_NATIVE_FIPS202_X3 in your backend and define the inline wrapper + * mld_keccak_f1600_x3_native() to forward to your implementation. + */ +#if defined(MLD_USE_NATIVE_FIPS202_X3) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) + ensures(return_value == MLD_NATIVE_FUNC_FALLBACK || return_value == MLD_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLD_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 3)) +); +#endif /* MLD_USE_NATIVE_FIPS202_X3 */ /* * Native x4 XOR bytes and extract bytes interface. diff --git a/mldsa/src/poly.c b/mldsa/src/poly.c index 4153d80b8d..6974442652 100644 --- a/mldsa/src/poly.c +++ b/mldsa/src/poly.c @@ -685,6 +685,7 @@ void mld_poly_uniform(mld_poly *a, const uint8_t seed[MLDSA_SEEDBYTES + 2]) #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_4x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, mld_poly *vec3, @@ -749,6 +750,70 @@ void mld_poly_uniform_4x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, mld_zeroize(buf, sizeof(buf)); } +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandA path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE128 XOF, same 2-byte (x,y) + * nonce seed layout (set by the caller), same mld_rej_uniform mod-q rejection + * loop; only the batch width differs (3 lanes instead of 4). */ +MLD_INTERNAL_API +void mld_poly_uniform_3x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, + uint8_t seed[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLD_ALIGN uint8_t + buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_NBLOCKS * MLD_STREAM128_BLOCKBYTES)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[3]; + mld_xof128_x3_ctx state; + unsigned buflen; + + mld_xof128_x3_init(&state); + mld_xof128_x3_absorb(&state, seed, MLDSA_SEEDBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLD_POLY_UNIFORM_NBLOCKS. + * This should generate the matrix entries with high probability. + */ + + mld_xof128_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_NBLOCKS, &state); + buflen = MLD_POLY_UNIFORM_NBLOCKS * MLD_STREAM128_BLOCKBYTES; + ctr[0] = mld_rej_uniform(vec0->coeffs, MLDSA_N, 0, buf[0], buflen); + ctr[1] = mld_rej_uniform(vec1->coeffs, MLDSA_N, 0, buf[1], buflen); + ctr[2] = mld_rej_uniform(vec2->coeffs, MLDSA_N, 0, buf[2], buflen); + + /* + * So long as not all matrix entries have been generated, squeeze + * one more block a time until we're done. + */ + buflen = MLD_STREAM128_BLOCKBYTES; + while (ctr[0] < MLDSA_N || ctr[1] < MLDSA_N || ctr[2] < MLDSA_N) + __loop__( + assigns(ctr, state, object_whole(buf), + memory_slice(vec0, sizeof(mld_poly)), memory_slice(vec1, sizeof(mld_poly)), + memory_slice(vec2, sizeof(mld_poly))) + invariant(ctr[0] <= MLDSA_N && ctr[1] <= MLDSA_N && ctr[2] <= MLDSA_N) + invariant(array_bound(vec0->coeffs, 0, ctr[0], 0, MLDSA_Q)) + invariant(array_bound(vec1->coeffs, 0, ctr[1], 0, MLDSA_Q)) + invariant(array_bound(vec2->coeffs, 0, ctr[2], 0, MLDSA_Q))) + { + mld_xof128_x3_squeezeblocks(buf, 1, &state); + ctr[0] = mld_rej_uniform(vec0->coeffs, MLDSA_N, ctr[0], buf[0], buflen); + ctr[1] = mld_rej_uniform(vec1->coeffs, MLDSA_N, ctr[1], buf[1], buflen); + ctr[2] = mld_rej_uniform(vec2->coeffs, MLDSA_N, ctr[2], buf[2], buflen); + } + mld_xof128_x3_release(&state); + + mld_assert_bound(vec0->coeffs, MLDSA_N, 0, MLDSA_Q); + mld_assert_bound(vec1->coeffs, MLDSA_N, 0, MLDSA_Q); + mld_assert_bound(vec2->coeffs, MLDSA_N, 0, MLDSA_Q); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); +} +#endif /* MLD_CONFIG_USE_X3 */ + #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ diff --git a/mldsa/src/poly.h b/mldsa/src/poly.h index 91cc426b76..faa1e804cc 100644 --- a/mldsa/src/poly.h +++ b/mldsa/src/poly.h @@ -265,6 +265,7 @@ __contract__( #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_4x MLD_NAMESPACE(poly_uniform_4x) /** * Generate four polynomials using rejection sampling on (pseudo-)uniformly @@ -298,6 +299,39 @@ __contract__( ensures(array_bound(vec2->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) ensures(array_bound(vec3->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_3x MLD_NAMESPACE(poly_uniform_3x) +/** + * Generate three polynomials using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. 3-way variant of + * mld_poly_uniform_4x, used with MLD_CONFIG_USE_X3, used by the x3 ExpandA + * path. + * + * @spec{Implements @[FIPS204, Algorithm 30, RejNTTPoly] (three-way batched).} + * + * @param[out] vec0 Pointer to first polynomial to be sampled. + * @param[out] vec1 Pointer to second polynomial to be sampled. + * @param[out] vec2 Pointer to third polynomial to be sampled. + * @param[in] seed Pointer to consecutive array of seed buffers of size + * MLDSA_SEEDBYTES + 2 each, plus padding for alignment. + */ +MLD_INTERNAL_API +void mld_poly_uniform_3x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, + uint8_t seed[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]) +__contract__( + requires(memory_no_alias(vec0, sizeof(mld_poly))) + requires(memory_no_alias(vec1, sizeof(mld_poly))) + requires(memory_no_alias(vec2, sizeof(mld_poly))) + requires(memory_no_alias(seed, 3 * MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2))) + assigns(memory_slice(vec0, sizeof(mld_poly))) + assigns(memory_slice(vec1, sizeof(mld_poly))) + assigns(memory_slice(vec2, sizeof(mld_poly))) + ensures(array_bound(vec0->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) + ensures(array_bound(vec1->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) + ensures(array_bound(vec2->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ diff --git a/mldsa/src/poly_kl.c b/mldsa/src/poly_kl.c index 95241c08a5..ab9af0754b 100644 --- a/mldsa/src/poly_kl.c +++ b/mldsa/src/poly_kl.c @@ -322,6 +322,7 @@ __contract__( } #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_eta_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_poly *r3, const uint8_t seed[MLDSA_CRHBYTES], @@ -405,6 +406,85 @@ void mld_poly_uniform_eta_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } + +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_eta_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandS path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE256 XOF, same 1-byte nonce + * seed layout, same mld_rej_eta rejection loop; only the batch width differs + * (3 lanes instead of 4). */ +MLD_INTERNAL_API +void mld_poly_uniform_eta_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], uint8_t nonce0, + uint8_t nonce1, uint8_t nonce2) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLD_ALIGN uint8_t buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_ETA_NBLOCKS * + MLD_STREAM256_BLOCKBYTES)]; + + MLD_ALIGN uint8_t extseed[3][MLD_ALIGN_UP(MLDSA_CRHBYTES + 2)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[3]; + mld_xof256_x3_ctx state; + unsigned buflen; + + mld_memcpy(extseed[0], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[1], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[2], seed, MLDSA_CRHBYTES); + extseed[0][MLDSA_CRHBYTES] = nonce0; + extseed[1][MLDSA_CRHBYTES] = nonce1; + extseed[2][MLDSA_CRHBYTES] = nonce2; + extseed[0][MLDSA_CRHBYTES + 1] = 0; + extseed[1][MLDSA_CRHBYTES + 1] = 0; + extseed[2][MLDSA_CRHBYTES + 1] = 0; + + mld_xof256_x3_init(&state); + mld_xof256_x3_absorb(&state, extseed, MLDSA_CRHBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLD_POLY_UNIFORM_ETA_NBLOCKS. + * This should generate the coefficients with high probability. + */ + mld_xof256_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_ETA_NBLOCKS, &state); + buflen = MLD_POLY_UNIFORM_ETA_NBLOCKS * MLD_STREAM256_BLOCKBYTES; + + ctr[0] = mld_rej_eta(r0->coeffs, MLDSA_N, 0, buf[0], buflen); + ctr[1] = mld_rej_eta(r1->coeffs, MLDSA_N, 0, buf[1], buflen); + ctr[2] = mld_rej_eta(r2->coeffs, MLDSA_N, 0, buf[2], buflen); + + /* + * So long as not all entries have been generated, squeeze + * one more block at a time until we're done. + */ + buflen = MLD_STREAM256_BLOCKBYTES; + while (ctr[0] < MLDSA_N || ctr[1] < MLDSA_N || ctr[2] < MLDSA_N) + __loop__( + assigns(ctr, state, memory_slice(r0, sizeof(mld_poly)), + memory_slice(r1, sizeof(mld_poly)), memory_slice(r2, sizeof(mld_poly)), + object_whole(buf[0]), object_whole(buf[1]), object_whole(buf[2])) + invariant(ctr[0] <= MLDSA_N && ctr[1] <= MLDSA_N && ctr[2] <= MLDSA_N) + invariant(array_abs_bound(r0->coeffs, 0, ctr[0], MLDSA_ETA + 1)) + invariant(array_abs_bound(r1->coeffs, 0, ctr[1], MLDSA_ETA + 1)) + invariant(array_abs_bound(r2->coeffs, 0, ctr[2], MLDSA_ETA + 1))) + { + mld_xof256_x3_squeezeblocks(buf, 1, &state); + ctr[0] = mld_rej_eta(r0->coeffs, MLDSA_N, ctr[0], buf[0], buflen); + ctr[1] = mld_rej_eta(r1->coeffs, MLDSA_N, ctr[1], buf[1], buflen); + ctr[2] = mld_rej_eta(r2->coeffs, MLDSA_N, ctr[2], buf[2], buflen); + } + + mld_xof256_x3_release(&state); + + mld_assert_abs_bound(r0->coeffs, MLDSA_N, MLDSA_ETA + 1); + mld_assert_abs_bound(r1->coeffs, MLDSA_N, MLDSA_ETA + 1); + mld_assert_abs_bound(r2->coeffs, MLDSA_N, MLDSA_ETA + 1); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); + mld_zeroize(extseed, sizeof(extseed)); +} +#endif /* MLD_CONFIG_USE_X3 */ #else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ MLD_INTERNAL_API @@ -470,8 +550,12 @@ void mld_poly_uniform_eta(mld_poly *r, const uint8_t seed[MLDSA_CRHBYTES], ((MLDSA_POLYZ_PACKEDBYTES + MLD_STREAM256_BLOCKBYTES - 1) / \ MLD_STREAM256_BLOCKBYTES) -#if MLD_CONFIG_PARAMETER_SET == 65 || \ - defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ +/* The serial sampler is used by the default x4 ExpandMask for L=5, and by the + * x3 ExpandMask (MLD_CONFIG_USE_X3) for its 1-polynomial remainder (L=4 and + * L=7). */ +#if (MLD_CONFIG_PARAMETER_SET == 65 && !defined(MLD_CONFIG_USE_X3)) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && defined(MLD_CONFIG_USE_X3)) || \ + defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) MLD_INTERNAL_API void mld_poly_uniform_gamma1(mld_poly *a, const uint8_t seed[MLDSA_CRHBYTES], @@ -500,12 +584,15 @@ void mld_poly_uniform_gamma1(mld_poly *a, const uint8_t seed[MLDSA_CRHBYTES], mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } -#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_SERIAL_FIPS202_ONLY || \ - MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ +#endif /* (MLD_CONFIG_PARAMETER_SET == 65 && !MLD_CONFIG_USE_X3) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && MLD_CONFIG_USE_X3) || \ + MLD_CONFIG_SERIAL_FIPS202_ONLY || MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_gamma1_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_poly *r3, @@ -554,6 +641,54 @@ void mld_poly_uniform_gamma1_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } + +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_gamma1_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandMask path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE256 XOF, same 2-byte nonce + * layout, same mld_polyz_unpack; only the batch width differs (3 lanes). */ +MLD_INTERNAL_API +void mld_poly_uniform_gamma1_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], + uint16_t nonce0, uint16_t nonce1, + uint16_t nonce2) +{ + /* Temporary buffers for XOF output before unpacking */ + MLD_ALIGN uint8_t buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_GAMMA1_NBLOCKS * + MLD_STREAM256_BLOCKBYTES)]; + + MLD_ALIGN uint8_t extseed[3][MLD_ALIGN_UP(MLDSA_CRHBYTES + 2)]; + + mld_xof256_x3_ctx state; + + mld_memcpy(extseed[0], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[1], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[2], seed, MLDSA_CRHBYTES); + extseed[0][MLDSA_CRHBYTES] = (uint8_t)(nonce0 & 0xFF); + extseed[1][MLDSA_CRHBYTES] = (uint8_t)(nonce1 & 0xFF); + extseed[2][MLDSA_CRHBYTES] = (uint8_t)(nonce2 & 0xFF); + extseed[0][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce0 >> 8); + extseed[1][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce1 >> 8); + extseed[2][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce2 >> 8); + + mld_xof256_x3_init(&state); + mld_xof256_x3_absorb(&state, extseed, MLDSA_CRHBYTES + 2); + mld_xof256_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_GAMMA1_NBLOCKS, &state); + + mld_polyz_unpack(r0, buf[0]); + mld_polyz_unpack(r1, buf[1]); + mld_polyz_unpack(r2, buf[2]); + mld_xof256_x3_release(&state); + + mld_assert_bound(r0->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + mld_assert_bound(r1->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + mld_assert_bound(r2->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); + mld_zeroize(extseed, sizeof(extseed)); +} +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ #endif /* !MLD_CONFIG_NO_SIGN_API */ diff --git a/mldsa/src/poly_kl.h b/mldsa/src/poly_kl.h index d31510ceb9..dfad54e7f2 100644 --- a/mldsa/src/poly_kl.h +++ b/mldsa/src/poly_kl.h @@ -72,6 +72,7 @@ __contract__( #if !defined(MLD_CONFIG_NO_KEYPAIR_API) #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_eta_4x MLD_NAMESPACE_KL(poly_uniform_eta_4x) /** * Sample four polynomials with uniformly random coefficients in @@ -111,6 +112,44 @@ __contract__( ensures(array_abs_bound(r2->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) ensures(array_abs_bound(r3->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_eta_3x MLD_NAMESPACE_KL(poly_uniform_eta_3x) +/** + * Sample three polynomials with uniformly random coefficients in + * [-MLDSA_ETA, MLDSA_ETA] by performing rejection sampling on the output + * stream from SHAKE256(seed|nonce_i). 3-way variant of + * mld_poly_uniform_eta_4x, used with MLD_CONFIG_USE_X3, used by the x3 + * ExpandS path. + * + * @spec{Implements @[FIPS204, Algorithm 31, RejBoundedPoly] (three-way + * batched).} + * + * @param[out] r0 Pointer to first output polynomial. + * @param[out] r1 Pointer to second output polynomial. + * @param[out] r2 Pointer to third output polynomial. + * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. + * @param nonce0 First nonce. + * @param nonce1 Second nonce. + * @param nonce2 Third nonce. + */ +MLD_INTERNAL_API +void mld_poly_uniform_eta_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], uint8_t nonce0, + uint8_t nonce1, uint8_t nonce2) +__contract__( + requires(memory_no_alias(r0, sizeof(mld_poly))) + requires(memory_no_alias(r1, sizeof(mld_poly))) + requires(memory_no_alias(r2, sizeof(mld_poly))) + requires(memory_no_alias(seed, MLDSA_CRHBYTES)) + assigns(memory_slice(r0, sizeof(mld_poly))) + assigns(memory_slice(r1, sizeof(mld_poly))) + assigns(memory_slice(r2, sizeof(mld_poly))) + ensures(array_abs_bound(r0->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) + ensures(array_abs_bound(r1->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) + ensures(array_abs_bound(r2->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ #if defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) @@ -139,8 +178,12 @@ __contract__( #endif /* !MLD_CONFIG_NO_KEYPAIR_API */ #if !defined(MLD_CONFIG_NO_SIGN_API) -#if MLD_CONFIG_PARAMETER_SET == 65 || \ - defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ +/* The serial sampler is used by the default x4 ExpandMask for L=5, and by the + * x3 ExpandMask (MLD_CONFIG_USE_X3) for its 1-polynomial remainder (L=4 and + * L=7). */ +#if (MLD_CONFIG_PARAMETER_SET == 65 && !defined(MLD_CONFIG_USE_X3)) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && defined(MLD_CONFIG_USE_X3)) || \ + defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) #define mld_poly_uniform_gamma1 MLD_NAMESPACE_KL(poly_uniform_gamma1) /** @@ -164,11 +207,14 @@ __contract__( assigns(memory_slice(a, sizeof(mld_poly))) ensures(array_bound(a->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ); -#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_SERIAL_FIPS202_ONLY || \ - MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ +#endif /* (MLD_CONFIG_PARAMETER_SET == 65 && !MLD_CONFIG_USE_X3) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && MLD_CONFIG_USE_X3) || \ + MLD_CONFIG_SERIAL_FIPS202_ONLY || MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_gamma1_4x MLD_NAMESPACE_KL(poly_uniform_gamma1_4x) /** * Sample four polynomials with uniformly random coefficients in @@ -209,6 +255,47 @@ __contract__( ensures(array_bound(r2->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ensures(array_bound(r3->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_gamma1_3x MLD_NAMESPACE_KL(poly_uniform_gamma1_3x) +/** + * Sample three polynomials with uniformly random coefficients in + * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1] by unpacking output streams of + * SHAKE256(seed|nonce_i). 3-way variant of + * mld_poly_uniform_gamma1_4x, used with MLD_CONFIG_USE_X3, used by the x3 + * ExpandMask path. Byte-for-byte identical per-lane behaviour to the x4 + * sampler: same SHAKE256 XOF, same 2-byte nonce layout, same mld_polyz_unpack; + * only the batch width differs (3 lanes instead of 4). + * + * @spec{Partially implements @[FIPS204, Algorithm 34, ExpandMask] (three-way + * batched).} + * + * @param[out] r0 Pointer to first output polynomial. + * @param[out] r1 Pointer to second output polynomial. + * @param[out] r2 Pointer to third output polynomial. + * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. + * @param nonce0 First 16-bit nonce. + * @param nonce1 Second 16-bit nonce. + * @param nonce2 Third 16-bit nonce. + */ +MLD_INTERNAL_API +void mld_poly_uniform_gamma1_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], + uint16_t nonce0, uint16_t nonce1, + uint16_t nonce2) +__contract__( + requires(memory_no_alias(r0, sizeof(mld_poly))) + requires(memory_no_alias(r1, sizeof(mld_poly))) + requires(memory_no_alias(r2, sizeof(mld_poly))) + requires(memory_no_alias(seed, MLDSA_CRHBYTES)) + assigns(memory_slice(r0, sizeof(mld_poly))) + assigns(memory_slice(r1, sizeof(mld_poly))) + assigns(memory_slice(r2, sizeof(mld_poly))) + ensures(array_bound(r0->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) + ensures(array_bound(r1->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) + ensures(array_bound(r2->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ #endif /* !MLD_CONFIG_NO_SIGN_API */ diff --git a/mldsa/src/polyvec.c b/mldsa/src/polyvec.c index be13ae3f91..69a8ba33df 100644 --- a/mldsa/src/polyvec.c +++ b/mldsa/src/polyvec.c @@ -17,6 +17,10 @@ #include "debug.h" #include "polyvec_lazy.h" +#if defined(MLD_CONFIG_USE_X3) +#include "ct.h" /* mld_zeroize, for the x3 ExpandMask scratch-lane remainder */ +#endif + /* This namespacing is not done at the top to avoid a naming conflict * with native backends, which are currently not yet namespaced. */ #define mld_polyvecl_pointwise_acc_montgomery_c \ @@ -44,7 +48,44 @@ void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, { mld_poly_uniform_gamma1(&v->vec[i], seed, (uint16_t)(kappa + i)); } -#else /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#elif defined(MLD_CONFIG_USE_X3) + /* + * x3 ExpandMask (MLD_CONFIG_USE_X3): sample the L + * mask polynomials 3 at a time via mld_poly_uniform_gamma1_3x. The nonce + * assigned to each polynomial is IDENTICAL to the x4 path below (poly i uses + * nonce kappa + i), so the sampled vector is byte-for-byte the same; only the + * batch width (3 vs 4) differs. Any sub-3 remainder is sampled serially + * (L=4, L=7) or via a scratch 3rd lane with a throwaway nonce (L=5). + */ +#if MLDSA_L == 4 + /* L = 4 = 1 + 3: one serial poly + one 3-way batch. */ + mld_poly_uniform_gamma1(&v->vec[0], seed, kappa); + mld_poly_uniform_gamma1_3x(&v->vec[1], &v->vec[2], &v->vec[3], seed, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2), + (uint16_t)(kappa + 3)); +#elif MLDSA_L == 5 + /* L = 5 = 3 + 2: two 3-way batches, the 3rd lane of the 2nd batch is a + * throwaway (scratch poly, nonce 0xFF, zeroized after). */ + { + MLD_ALIGN mld_poly scratch; + mld_poly_uniform_gamma1_3x(&v->vec[0], &v->vec[1], &v->vec[2], seed, kappa, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2)); + mld_poly_uniform_gamma1_3x(&v->vec[3], &v->vec[4], &scratch, seed, + (uint16_t)(kappa + 3), (uint16_t)(kappa + 4), + 0xFF /* irrelevant */); + mld_zeroize(&scratch, sizeof(scratch)); + } +#elif MLDSA_L == 7 + /* L = 7 = 1 + 3 + 3: one serial poly + two 3-way batches. */ + mld_poly_uniform_gamma1(&v->vec[0], seed, kappa); + mld_poly_uniform_gamma1_3x(&v->vec[1], &v->vec[2], &v->vec[3], seed, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2), + (uint16_t)(kappa + 3)); + mld_poly_uniform_gamma1_3x(&v->vec[4], &v->vec[5], &v->vec[6], seed, + (uint16_t)(kappa + 4), (uint16_t)(kappa + 5), + (uint16_t)(kappa + 6)); +#endif /* MLDSA_L == 7 */ +#else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ #if MLDSA_L == 4 mld_poly_uniform_gamma1_4x(&v->vec[0], &v->vec[1], &v->vec[2], &v->vec[3], seed, kappa, (uint16_t)(kappa + 1), @@ -63,7 +104,7 @@ void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, seed, (uint16_t)(kappa + 3), (uint16_t)(kappa + 4), (uint16_t)(kappa + 5), (uint16_t)(kappa + 6)); #endif /* MLDSA_L == 7 */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ mld_assert_bound_2d(v->vec, MLDSA_L, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); diff --git a/mldsa/src/polyvec_lazy.c b/mldsa/src/polyvec_lazy.c index dab3ce96a9..3e06ba2676 100644 --- a/mldsa/src/polyvec_lazy.c +++ b/mldsa/src/polyvec_lazy.c @@ -76,6 +76,54 @@ void mld_polyvec_matrix_expand_eager(mld_polymat_eager *mat, } #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if defined(MLD_CONFIG_USE_X3) + /* + * x3 ExpandA (MLD_CONFIG_USE_X3): sample 3 matrix + * entries at a time via mld_poly_uniform_3x. The (x, y) nonce assignment and + * sampling order are identical to the x4 path below, so the sampled matrix A + * is byte-for-byte the same; only the batch width (3 vs 4) differs. Any + * remainder (K*L % 3) is sampled individually by the shared serial tail loop + * below. + */ + for (i = 0; i < (MLDSA_K * MLDSA_L / 3) * 3; i += 3) + __loop__( + assigns(i, j, object_whole(seed_ext), memory_slice(mat, sizeof(mld_polymat_eager))) + invariant(i <= (MLDSA_K * MLDSA_L / 3) * 3 && i % 3 == 0) + /* vectors 0 .. i / MLDSA_L are completely sampled */ + invariant(forall(k1, 0, i / MLDSA_L, forall(l1, 0, MLDSA_L, + array_bound(mat->vec[k1].vec[l1].coeffs, 0, MLDSA_N, 0, MLDSA_Q)))) + /* last vector is sampled up to i % MLDSA_L */ + invariant(forall(k2, i / MLDSA_L, i / MLDSA_L + 1, forall(l2, 0, i % MLDSA_L, + array_bound(mat->vec[k2].vec[l2].coeffs, 0, MLDSA_N, 0, MLDSA_Q)))) + decreases((MLDSA_K * MLDSA_L / 3) * 3 - i) + ) + { + for (j = 0; j < 3; j++) + __loop__( + assigns(j, object_whole(seed_ext)) + invariant(j <= 3) + decreases(3 - j) + ) + { + uint8_t x = (uint8_t)((i + j) / MLDSA_L); + uint8_t y = (uint8_t)((i + j) % MLDSA_L); + + seed_ext[j][MLDSA_SEEDBYTES + 0] = y; + seed_ext[j][MLDSA_SEEDBYTES + 1] = x; + } + + mld_poly_uniform_3x(&mat->vec[i / MLDSA_L].vec[i % MLDSA_L], + &mat->vec[(i + 1) / MLDSA_L].vec[(i + 1) % MLDSA_L], + &mat->vec[(i + 2) / MLDSA_L].vec[(i + 2) % MLDSA_L], + seed_ext); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[i / MLDSA_L].vec[i % MLDSA_L]); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[(i + 1) / MLDSA_L].vec[(i + 1) % MLDSA_L]); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[(i + 2) / MLDSA_L].vec[(i + 2) % MLDSA_L]); + } +#else /* MLD_CONFIG_USE_X3 */ /* Sample 4 matrix entries a time. */ for (i = 0; i < (MLDSA_K * MLDSA_L / 4) * 4; i += 4) __loop__( @@ -118,6 +166,7 @@ void mld_polyvec_matrix_expand_eager(mld_polymat_eager *mat, mld_poly_permute_bitrev_to_custom_optional( &mat->vec[(i + 3) / MLDSA_L].vec[(i + 3) % MLDSA_L]); } +#endif /* !MLD_CONFIG_USE_X3 */ #else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ i = 0; #endif /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/sign.c b/mldsa/src/sign.c index 48dac1184b..c597a0aa07 100644 --- a/mldsa/src/sign.c +++ b/mldsa/src/sign.c @@ -227,7 +227,67 @@ __contract__( mld_poly_uniform_eta(&s1->vec[i], seed, (uint8_t)i); mld_polyeta_pack(s1_packed + i * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[i]); } -#else /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#elif defined(MLD_CONFIG_USE_X3) +/* x3 ExpandS (MLD_CONFIG_USE_X3): same nonce assignment and the same + * s2-first, s1-as-scratch order as the 4-way path below, with batches of 3. + * A lane that is not needed carries nonce 0xFF and writes an s1 slot that is + * sampled afterwards. */ +#if MLD_CONFIG_PARAMETER_SET == 44 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 4, 5, 6); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[3], &s1->vec[1], seed, 7, 3, + 0xFF /* irrelevant */); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#elif MLD_CONFIG_PARAMETER_SET == 65 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 5, 6, 7); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 8, 9, + 10); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[3], &s1->vec[4], &s1->vec[0], seed, 3, 4, + 0xFF /* irrelevant */); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_polyeta_pack(s1_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[4]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#elif MLD_CONFIG_PARAMETER_SET == 87 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 7, 8, 9); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 10, 11, + 12); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[6], seed, 13, 14, + 6); + mld_polyeta_pack(s2_packed + 6 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 7 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 6 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[6]); + mld_poly_uniform_eta_3x(&s1->vec[3], &s1->vec[4], &s1->vec[5], seed, 3, 4, 5); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_polyeta_pack(s1_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[4]); + mld_polyeta_pack(s1_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[5]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#endif /* MLD_CONFIG_PARAMETER_SET == 87 */ +#else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ /* Batches keep the nonce grouping of the serial path; s2 and don't-care * outputs go to s1 slots that are yet to be sampled. */ #if MLD_CONFIG_PARAMETER_SET == 44 @@ -288,7 +348,7 @@ __contract__( mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); #endif /* MLD_CONFIG_PARAMETER_SET == 87 */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ } /** diff --git a/mldsa/src/symmetric.h b/mldsa/src/symmetric.h index 4bbd4f19c0..5dfe642f36 100644 --- a/mldsa/src/symmetric.h +++ b/mldsa/src/symmetric.h @@ -11,7 +11,10 @@ #include MLD_FIPS202_HEADER_FILE #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) #include MLD_FIPS202X4_HEADER_FILE +#if defined(MLD_CONFIG_USE_X3) +#include "fips202/fips202x3.h" #endif +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ #define MLD_STREAM128_BLOCKBYTES SHAKE128_RATE #define MLD_STREAM256_BLOCKBYTES SHAKE256_RATE @@ -65,4 +68,28 @@ (NBLOCKS), (CTX)) #define mld_xof128_x4_release(CTX) mld_shake128x4_release((CTX)) +#if defined(MLD_CONFIG_USE_X3) +/* + * 3-way XOF, used with MLD_CONFIG_USE_X3. Mirrors the x4 macros above but + * 3-wide. SHAKE128 backs the x3 ExpandA path (mld_poly_uniform_3x); SHAKE256 + * backs the x3 ExpandS and ExpandMask paths (mld_poly_uniform_eta_3x, + * mld_poly_uniform_gamma1_3x). + */ +#define mld_xof128_x3_ctx mld_shake128x3ctx +#define mld_xof128_x3_init(CTX) mld_shake128x3_init((CTX)) +#define mld_xof128_x3_absorb(CTX, IN, INBYTES) \ + mld_shake128x3_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (INBYTES)) +#define mld_xof128_x3_squeezeblocks(BUF, NBLOCKS, CTX) \ + mld_shake128x3_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (NBLOCKS), (CTX)) +#define mld_xof128_x3_release(CTX) mld_shake128x3_release((CTX)) + +#define mld_xof256_x3_ctx mld_shake256x3ctx +#define mld_xof256_x3_init(CTX) mld_shake256x3_init((CTX)) +#define mld_xof256_x3_absorb(CTX, IN, INBYTES) \ + mld_shake256x3_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (INBYTES)) +#define mld_xof256_x3_squeezeblocks(BUF, NBLOCKS, CTX) \ + mld_shake256x3_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (NBLOCKS), (CTX)) +#define mld_xof256_x3_release(CTX) mld_shake256x3_release((CTX)) +#endif /* MLD_CONFIG_USE_X3 */ + #endif /* !MLD_SYMMETRIC_H */ diff --git a/proofs/cbmc/dummy_backend_fips202_x3.h b/proofs/cbmc/dummy_backend_fips202_x3.h new file mode 100644 index 0000000000..24a4db5e71 --- /dev/null +++ b/proofs/cbmc/dummy_backend_fips202_x3.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DUMMY_FIPS202X3_BACKEND_H +#define MLD_DUMMY_FIPS202X3_BACKEND_H + + +#define MLD_USE_NATIVE_FIPS202_X3 + +#include "../../mldsa/src/fips202/native/api.h" + +#endif /* !MLD_DUMMY_FIPS202X3_BACKEND_H */ diff --git a/proofs/cbmc/keccak_absorb_once_x3/Makefile b/proofs/cbmc/keccak_absorb_once_x3/Makefile new file mode 100644 index 0000000000..2bcf0842c0 --- /dev/null +++ b/proofs/cbmc/keccak_absorb_once_x3/Makefile @@ -0,0 +1,57 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_absorb_once_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_absorb_once_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS+=mld_keccakf1600x3_xor_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccak_absorb_once_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c b/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c new file mode 100644 index 0000000000..3d60cc285a --- /dev/null +++ b/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c @@ -0,0 +1,20 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void mld_keccak_absorb_once_x3(uint64_t *s, unsigned r, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen, uint8_t p); + +void harness(void) +{ + uint64_t *s; + unsigned r; + const uint8_t *in0, *in1, *in2; + size_t inlen; + uint8_t p; + mld_keccak_absorb_once_x3(s, r, in0, in1, in2, inlen, p); +} diff --git a/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile new file mode 100644 index 0000000000..573724cadd --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mlkem-native project authors +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid + +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 -DMLD_CONFIG_FIPS202_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h\"" -DMLD_CHECK_APIS -D__ARM_FEATURE_SHA3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_native +USE_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c new file mode 100644 index 0000000000..ccd48eab0a --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +int mld_keccak_f1600_x3_native(uint64_t *state); + +void harness(void) +{ + uint64_t *s; + int t = mld_keccak_f1600_x3_native(s); +} diff --git a/proofs/cbmc/keccak_squeezeblocks_x3/Makefile b/proofs/cbmc/keccak_squeezeblocks_x3/Makefile new file mode 100644 index 0000000000..bd1c4e804d --- /dev/null +++ b/proofs/cbmc/keccak_squeezeblocks_x3/Makefile @@ -0,0 +1,57 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_squeezeblocks_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_squeezeblocks_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes +USE_FUNCTION_CONTRACTS+=mld_keccakf1600x3_permute +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccak_squeezeblocks_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c b/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c new file mode 100644 index 0000000000..5e37766f37 --- /dev/null +++ b/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void mld_keccak_squeezeblocks_x3(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, uint64_t *s, unsigned r); + +void harness(void) +{ + uint8_t *out0, *out1, *out2; + size_t nblocks; + uint64_t *s; + unsigned r; + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, s, r); +} diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile b/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile new file mode 100644 index 0000000000..b69eb38433 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_extract_bytes_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_extract_bytes + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes_c +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccakf1600x3_extract_bytes + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c b/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c new file mode 100644 index 0000000000..fcd2eab76f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c @@ -0,0 +1,13 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *state; + unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_extract_bytes(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile b/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile new file mode 100644 index 0000000000..dda7a71110 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_extract_bytes_c_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_extract_bytes_c + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes_c +USE_FUNCTION_CONTRACTS=mld_keccakf1600_extract_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_extract_bytes_c + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c b/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c new file mode 100644 index 0000000000..ae2c7d0d99 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void mld_keccakf1600x3_extract_bytes_c(uint64_t *state, unsigned char *data0, + unsigned char *data1, + unsigned char *data2, unsigned offset, + unsigned length); + +void harness(void) +{ + uint64_t *state; + unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_extract_bytes_c(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_permute/Makefile b/proofs/cbmc/keccakf1600x3_permute/Makefile new file mode 100644 index 0000000000..54af6a8a23 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_permute_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_permute + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS=mld_keccakf1600_permute +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_permute + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c b/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c new file mode 100644 index 0000000000..62aeefacc0 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c @@ -0,0 +1,10 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *s; + mld_keccakf1600x3_permute(s); +} diff --git a/proofs/cbmc/keccakf1600x3_permute_native/Makefile b/proofs/cbmc/keccakf1600x3_permute_native/Makefile new file mode 100644 index 0000000000..af20c36e8b --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute_native/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_permute_native_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_permute_native + +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 -DMLD_CONFIG_FIPS202_BACKEND_FILE="\"dummy_backend_fips202_x3.h\"" -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_native +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_permute_native + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c b/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c new file mode 100644 index 0000000000..62aeefacc0 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c @@ -0,0 +1,10 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *s; + mld_keccakf1600x3_permute(s); +} diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile b/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile new file mode 100644 index 0000000000..095a7f4e81 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_xor_bytes_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_xor_bytes + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes_c +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccakf1600x3_xor_bytes + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c b/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c new file mode 100644 index 0000000000..68a9ced14f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c @@ -0,0 +1,13 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *state; + const unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_xor_bytes(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile b/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile new file mode 100644 index 0000000000..561a01c4eb --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_xor_bytes_c_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_xor_bytes_c + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes_c +USE_FUNCTION_CONTRACTS=mld_keccakf1600_xor_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_xor_bytes_c + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c b/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c new file mode 100644 index 0000000000..3f9e1dca6f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void mld_keccakf1600x3_xor_bytes_c(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length); + +void harness(void) +{ + uint64_t *state; + const unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_xor_bytes_c(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/mldsa_native_config_cbmc.h b/proofs/cbmc/mldsa_native_config_cbmc.h index 8c5e3cf7f5..316aef6f16 100644 --- a/proofs/cbmc/mldsa_native_config_cbmc.h +++ b/proofs/cbmc/mldsa_native_config_cbmc.h @@ -414,6 +414,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/proofs/cbmc/poly_uniform_3x/Makefile b/proofs/cbmc/poly_uniform_3x/Makefile new file mode 100644 index 0000000000..684001abfc --- /dev/null +++ b/proofs/cbmc/poly_uniform_3x/Makefile @@ -0,0 +1,63 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_3x +USE_FUNCTION_CONTRACTS=mld_shake128x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake128x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_rej_uniform +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c b/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c new file mode 100644 index 0000000000..b1162fe39c --- /dev/null +++ b/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_poly *r0; + mld_poly *r1; + mld_poly *r2; + uint8_t (*seed)[MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + + mld_poly_uniform_3x(r0, r1, r2, seed); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/poly_uniform_eta_3x/Makefile b/proofs/cbmc/poly_uniform_eta_3x/Makefile new file mode 100644 index 0000000000..99c8697c1e --- /dev/null +++ b/proofs/cbmc/poly_uniform_eta_3x/Makefile @@ -0,0 +1,58 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_eta_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_eta_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_eta_3x +USE_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_rej_eta +USE_FUNCTION_CONTRACTS+=mld_zeroize +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_eta_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c b/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c new file mode 100644 index 0000000000..8b263fd19a --- /dev/null +++ b/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly_kl.h" + +void harness(void) +{ + mld_poly *r0, *r1, *r2; + const uint8_t *seed; + uint8_t n0, n1, n2; + mld_poly_uniform_eta_3x(r0, r1, r2, seed, n0, n1, n2); +} diff --git a/proofs/cbmc/poly_uniform_gamma1_3x/Makefile b/proofs/cbmc/poly_uniform_gamma1_3x/Makefile new file mode 100644 index 0000000000..c65ae6c6d5 --- /dev/null +++ b/proofs/cbmc/poly_uniform_gamma1_3x/Makefile @@ -0,0 +1,63 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_gamma1_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_gamma1_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_gamma1_3x +USE_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_polyz_unpack +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_gamma1_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c b/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c new file mode 100644 index 0000000000..88cb97a239 --- /dev/null +++ b/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly_kl.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_poly *r0, *r1, *r2; + const uint8_t *seed; + uint16_t n0, n1, n2; + mld_poly_uniform_gamma1_3x(r0, r1, r2, seed, n0, n1, n2); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/polyvec_matrix_expand_x3/Makefile b/proofs/cbmc/polyvec_matrix_expand_x3/Makefile new file mode 100644 index 0000000000..87fdc556c3 --- /dev/null +++ b/proofs/cbmc/polyvec_matrix_expand_x3/Makefile @@ -0,0 +1,62 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyvec_matrix_expand_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyvec_matrix_expand_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/polyvec_lazy.c + +CHECK_FUNCTION_CONTRACTS=mld_polyvec_matrix_expand +ifdef MLD_CONFIG_REDUCE_RAM +USE_FUNCTION_CONTRACTS= +else +USE_FUNCTION_CONTRACTS=mld_poly_uniform_3x +USE_FUNCTION_CONTRACTS+=mld_poly_permute_bitrev_to_custom_optional +USE_FUNCTION_CONTRACTS+=mld_polymat_expand_entry +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3_no_bv_extract +CBMC_DEFAULT_CONFIG = LP64:z3_no_bv_extract + +FUNCTION_NAME = polyvec_matrix_expand_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 12 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c b/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c new file mode 100644 index 0000000000..586ecbe493 --- /dev/null +++ b/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "polyvec_lazy.h" + +void harness(void) +{ + mld_polymat *mat; + uint8_t *rho; + + mld_polyvec_matrix_expand(mat, rho); +} diff --git a/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile b/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile new file mode 100644 index 0000000000..a4a6227894 --- /dev/null +++ b/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile @@ -0,0 +1,68 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyvecl_uniform_gamma1_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyvecl_uniform_gamma1_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/polyvec.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_polyvecl_uniform_gamma1 +USE_FUNCTION_CONTRACTS=mld_poly_uniform_gamma1_3x +# L=5 (ML-DSA-65) uses a zeroized scratch lane; L=4 and L=7 sample one +# polynomial serially. +ifeq ($(MLD_CONFIG_PARAMETER_SET),65) +USE_FUNCTION_CONTRACTS+=mld_zeroize +else +USE_FUNCTION_CONTRACTS+=mld_poly_uniform_gamma1 +endif +endif + +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = polyvecl_uniform_gamma1_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c b/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c new file mode 100644 index 0000000000..c8e4a799cc --- /dev/null +++ b/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c @@ -0,0 +1,15 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "polyvec.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_polyvecl *v; + const uint8_t *seed; + uint16_t kappa; + + mld_polyvecl_uniform_gamma1(v, seed, kappa); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/sample_pack_s1_s2_x3/Makefile b/proofs/cbmc/sample_pack_s1_s2_x3/Makefile new file mode 100644 index 0000000000..d798a50787 --- /dev/null +++ b/proofs/cbmc/sample_pack_s1_s2_x3/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = sample_pack_s1_s2_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = sample_pack_s1_s2_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/sign.c + +CHECK_FUNCTION_CONTRACTS=mld_sample_pack_s1_s2 +USE_FUNCTION_CONTRACTS=mld_poly_uniform_eta_3x +USE_FUNCTION_CONTRACTS+=mld_polyeta_pack +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_sample_pack_s1_s2_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c b/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c new file mode 100644 index 0000000000..fa13980785 --- /dev/null +++ b/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "sign.h" + +static void mld_sample_pack_s1_s2( + uint8_t s1_packed[MLDSA_L * MLDSA_POLYETA_PACKEDBYTES], + uint8_t s2_packed[MLDSA_K * MLDSA_POLYETA_PACKEDBYTES], mld_polyvecl *s1, + const uint8_t seed[MLDSA_CRHBYTES]); + +void harness(void) +{ + uint8_t *s1_packed; + uint8_t *s2_packed; + uint8_t *seed; + mld_polyvecl *s1; + + mld_sample_pack_s1_s2(s1_packed, s2_packed, s1, seed); +} diff --git a/proofs/cbmc/shake128x3_absorb_once/Makefile b/proofs/cbmc/shake128x3_absorb_once/Makefile new file mode 100644 index 0000000000..7108e00d04 --- /dev/null +++ b/proofs/cbmc/shake128x3_absorb_once/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake128x3_absorb_once_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake128x3_absorb_once + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_shake128x3_absorb_once +USE_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake128x3_absorb_once + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c b/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c new file mode 100644 index 0000000000..85809eb70c --- /dev/null +++ b/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_shake128x3ctx *state; + const uint8_t *in0, *in1, *in2; + size_t inlen; + mld_shake128x3_absorb_once(state, in0, in1, in2, inlen); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/shake128x3_squeezeblocks/Makefile b/proofs/cbmc/shake128x3_squeezeblocks/Makefile new file mode 100644 index 0000000000..20fa7d47cf --- /dev/null +++ b/proofs/cbmc/shake128x3_squeezeblocks/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake128x3_squeezeblocks_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake128x3_squeezeblocks + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_shake128x3_squeezeblocks +USE_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake128x3_squeezeblocks + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c b/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c new file mode 100644 index 0000000000..a7e6746760 --- /dev/null +++ b/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + uint8_t *out0, *out1, *out2; + size_t nblocks; + mld_shake128x3ctx *state; + mld_shake128x3_squeezeblocks(out0, out1, out2, nblocks, state); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/shake256x3_absorb_once/Makefile b/proofs/cbmc/shake256x3_absorb_once/Makefile new file mode 100644 index 0000000000..ac3be69454 --- /dev/null +++ b/proofs/cbmc/shake256x3_absorb_once/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake256x3_absorb_once_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake256x3_absorb_once + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake256x3_absorb_once + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c b/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c new file mode 100644 index 0000000000..f84a1242a4 --- /dev/null +++ b/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ + mld_shake256x3ctx *state; + const uint8_t *in0, *in1, *in2; + size_t inlen; + mld_shake256x3_absorb_once(state, in0, in1, in2, inlen); +} diff --git a/proofs/cbmc/shake256x3_squeezeblocks/Makefile b/proofs/cbmc/shake256x3_squeezeblocks/Makefile new file mode 100644 index 0000000000..75efcf07cc --- /dev/null +++ b/proofs/cbmc/shake256x3_squeezeblocks/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake256x3_squeezeblocks_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake256x3_squeezeblocks + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake256x3_squeezeblocks + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c b/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c new file mode 100644 index 0000000000..b6cf9a778a --- /dev/null +++ b/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ + uint8_t *out0, *out1, *out2; + size_t nblocks; + mld_shake256x3ctx *state; + mld_shake256x3_squeezeblocks(out0, out1, out2, nblocks, state); +} diff --git a/proofs/hol_light/README.md b/proofs/hol_light/README.md index 6f0edfc4f4..42ebb6578a 100644 --- a/proofs/hol_light/README.md +++ b/proofs/hol_light/README.md @@ -219,6 +219,7 @@ All routines listed below have been proven correct, memory-safe, and secret-inde * Keccak-F1600 using lazy rotations[^HYBRID]: [keccak_f1600_x1_scalar_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x1_scalar_aarch64_asm.S) * Keccak-F1600 using v8.4-A SHA3 instructions: [keccak_f1600_x1_v84a_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x1_v84a_aarch64_asm.S) * 2-fold Keccak-F1600 using v8.4-A SHA3 instructions: [keccak_f1600_x2_v84a_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x2_v84a_aarch64_asm.S) + * 'Hybrid' 3-fold Keccak-F1600 using scalar and v8.4-A+SHA3 Neon instructions: [keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S) * 'Hybrid' 4-fold Keccak-F1600 using scalar and v8-A Neon instructions: [keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S) * 'Triple hybrid' 4-fold Keccak-F1600 using scalar, v8-A Neon and v8.4-A+SHA3 Neon instructions: [keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S) diff --git a/proofs/hol_light/aarch64/Makefile b/proofs/hol_light/aarch64/Makefile index b4a5254618..1af35214b8 100644 --- a/proofs/hol_light/aarch64/Makefile +++ b/proofs/hol_light/aarch64/Makefile @@ -68,6 +68,7 @@ OBJ = mldsa/mldsa_intt_aarch64_asm.o \ mldsa/keccak_f1600_x1_scalar_aarch64_asm.o \ mldsa/keccak_f1600_x1_v84a_aarch64_asm.o \ mldsa/keccak_f1600_x2_v84a_aarch64_asm.o \ + mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o \ mldsa/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.o \ mldsa/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.o \ mldsa/mldsa_polyz_unpack_17_aarch64_asm.o \ diff --git a/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..8d206ae486 --- /dev/null +++ b/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,642 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +#ifdef __APPLE__ +.global _mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +_mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: +#else +.global mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: +#endif + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x27, x26, x7 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x29, x0, x22 + eor x26, x29, x23 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x29, x4, x5 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x29, x30, x2 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x27, x26, x0, ror #63 + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x4, x4, x27 + eor x30, x30, x17 + rax1 v25.2d, v30.2d, v28.2d + eor x30, x30, x28 + eor x29, x29, x3 + rax1 v28.2d, v28.2d, v26.2d + eor x0, x0, x30, ror #63 + rax1 v26.2d, v26.2d, v29.2d + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + rax1 v29.2d, v29.2d, v27.2d + eor x23, x23, x30 + rax1 v27.2d, v27.2d, v30.2d + str x23, [sp, #0xd0] + eor x23, x14, x15 + eor v30.16b, v0.16b, v26.16b + eor x14, x14, x0 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x23, x23, x11 + eor x15, x15, x0 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x1, x1, x27 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x23, x23, x12 + eor x23, x23, x13 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x23, x23, x26, ror #63 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x26, x13, x0 + eor x13, x28, x23 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x28, x24, x30 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x24, x16, x23 + eor x16, x21, x30 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x21, x25, x30 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x30, x19, x23 + eor x19, x20, x23 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x20, x17, x23 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x17, x12, x0 + eor x0, x2, x27 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x2, x6, x29 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x5, x5, x27 + xar v9.2d, v22.2d, v29.2d, #0x3 + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x9, x9, x29 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x8, x8, x17, ror #2 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x7, x29 + xar v21.2d, v5.2d, v26.2d, #0x1c + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x7, x2, x5, ror #47 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x5, x17, x4, ror #25 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + bcax v9.16b, v9.16b, v11.16b, v10.16b + eor x28, x7, x28, ror #57 + bcax v10.16b, v15.16b, v12.16b, v16.16b + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bcax v11.16b, v16.16b, v13.16b, v12.16b + bic x22, x22, x24, ror #56 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bcax v13.16b, v13.16b, v15.16b, v14.16b + bic x20, x27, x20, ror #48 + bcax v14.16b, v14.16b, v16.16b, v15.16b + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x11, [sp, #0x8] + bcax v17.16b, v17.16b, v19.16b, v18.16b + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + bcax v18.16b, v18.16b, v20.16b, v19.16b + mov x24, #0x1 // =1 + bcax v19.16b, v19.16b, v21.16b, v20.16b + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + bcax v20.16b, v0.16b, v22.16b, v1.16b + bic x24, x29, x1, ror #44 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + bcax v22.16b, v22.16b, v24.16b, v23.16b + eor x1, x1, x4, ror #21 + bcax v23.16b, v23.16b, v0.16b, v24.16b + ldr x11, [x11] + bic x4, x21, x30, ror #57 + bcax v24.16b, v24.16b, v1.16b, v0.16b + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bcax v0.16b, v30.16b, v2.16b, v27.16b + bic x9, x14, x6, ror #5 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x1, x1, x11 + bcax v3.16b, v3.16b, v30.16b, v4.16b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + bic x26, x19, x14, ror #41 + str x30, [sp, #0x10] + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor v0.16b, v0.16b, v28.16b + eor x6, x27, x29, ror #41 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x29, x16, x28, ror #63 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x27, x27, x12, ror #5 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x26, x30, x21, ror #26 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + rax1 v25.2d, v30.2d, v28.2d + eor x16, x30, x16 + rax1 v28.2d, v28.2d, v26.2d + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + rax1 v26.2d, v26.2d, v29.2d + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + rax1 v29.2d, v29.2d, v27.2d + eor x19, x30, x19, ror #37 + rax1 v27.2d, v27.2d, v30.2d + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x26, x26, x0, ror #55 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + xar v2.2d, v12.2d, v29.2d, #0x15 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x27, x28, x27, ror #61 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x11, x0, x11, ror #50 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x21, x26, x1 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x22, x28, x22, ror #44 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x4, x26, x4, ror #54 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x26, x2, ror #61 + xar v9.2d, v22.2d, v29.2d, #0x3 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + xar v22.2d, v14.2d, v27.2d, #0x19 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x7, x7, x22, ror #25 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + xar v24.2d, v21.2d, v25.2d, #0x3e + bic x6, x19, x13, ror #42 + xar v21.2d, v5.2d, v26.2d, #0x1c + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x5, x21, x5, ror #21 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bcax v6.16b, v11.16b, v8.16b, v7.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x10, x16, x15, ror #31 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bcax v9.16b, v9.16b, v11.16b, v10.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x17, x10, x9, ror #47 + bcax v11.16b, v16.16b, v13.16b, v12.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x20, x4, x28, ror #2 + bcax v13.16b, v13.16b, v15.16b, v14.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v14.16b, v14.16b, v16.16b, v15.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x1, x1, x11, ror #57 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v17.16b, v17.16b, v19.16b, v18.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bcax v18.16b, v18.16b, v20.16b, v19.16b + cmp x25, #0x17 + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + bcax v20.16b, v0.16b, v22.16b, v1.16b + eor x1, x5, x28 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bcax v22.16b, v22.16b, v24.16b, v23.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bcax v23.16b, v23.16b, v0.16b, v24.16b + bic x27, x24, x9, ror #47 + bcax v24.16b, v24.16b, v1.16b, v0.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x13, x13, x9, ror #57 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v3.16b, v3.16b, v30.16b, v4.16b + bic x14, x14, x8, ror #5 + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + b.le Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml b/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml new file mode 100644 index 0000000000..84e46d9df7 --- /dev/null +++ b/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml @@ -0,0 +1,882 @@ +(* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0 + *) + +(* ========================================================================= *) +(* 3-way batch Keccak-f1600 scalar/vector hybrid code (single pass). *) +(* Derived from the x4 v84a proof: *) +(* keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml *) +(* ========================================================================= *) + +(* Load base theories for AArch64 from s2n-bignum *) +needs "s2n_bignum/arm/proofs/base.ml";; + +needs "mldsa_native/aarch64/proofs/keccak_utils.ml";; + +(**** print_literal_from_elf "aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o";; + ****) + +let keccak_f1600_x3_v84a_scalar_mc = define_assert_from_elf + "keccak_f1600_x3_v84a_scalar_mc" "aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o" +(*** BYTECODE START ***) +[ + 0xd10383ff; (* arm_SUB SP SP (rvalue (word 224)) *) + 0xa90353f3; (* arm_STP X19 X20 SP (Immediate_Offset (iword (&48))) *) + 0xa9045bf5; (* arm_STP X21 X22 SP (Immediate_Offset (iword (&64))) *) + 0xa90563f7; (* arm_STP X23 X24 SP (Immediate_Offset (iword (&80))) *) + 0xa9066bf9; (* arm_STP X25 X26 SP (Immediate_Offset (iword (&96))) *) + 0xa90773fb; (* arm_STP X27 X28 SP (Immediate_Offset (iword (&112))) *) + 0xa9087bfd; (* arm_STP X29 X30 SP (Immediate_Offset (iword (&128))) *) + 0x6d0927e8; (* arm_STP D8 D9 SP (Immediate_Offset (iword (&144))) *) + 0x6d0a2fea; (* arm_STP D10 D11 SP (Immediate_Offset (iword (&160))) *) + 0x6d0b37ec; (* arm_STP D12 D13 SP (Immediate_Offset (iword (&176))) *) + 0x6d0c3fee; (* arm_STP D14 D15 SP (Immediate_Offset (iword (&192))) *) + 0xaa0103fd; (* arm_MOV X29 X1 *) + 0xf90007fd; (* arm_STR X29 SP (Immediate_Offset (word 8)) *) + 0xf9000bfd; (* arm_STR X29 SP (Immediate_Offset (word 16)) *) + 0xf90003e0; (* arm_STR X0 SP (Immediate_Offset (word 0)) *) + 0x91032004; (* arm_ADD X4 X0 (rvalue (word 200)) *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b20; (* arm_TRN1 Q0 Q25 Q27 64 128 *) + 0x4edb6b21; (* arm_TRN2 Q1 Q25 Q27 64 128 *) + 0x4edc2b42; (* arm_TRN1 Q2 Q26 Q28 64 128 *) + 0x4edc6b43; (* arm_TRN2 Q3 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b24; (* arm_TRN1 Q4 Q25 Q27 64 128 *) + 0x4edb6b25; (* arm_TRN2 Q5 Q25 Q27 64 128 *) + 0x4edc2b46; (* arm_TRN1 Q6 Q26 Q28 64 128 *) + 0x4edc6b47; (* arm_TRN2 Q7 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b28; (* arm_TRN1 Q8 Q25 Q27 64 128 *) + 0x4edb6b29; (* arm_TRN2 Q9 Q25 Q27 64 128 *) + 0x4edc2b4a; (* arm_TRN1 Q10 Q26 Q28 64 128 *) + 0x4edc6b4b; (* arm_TRN2 Q11 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b2c; (* arm_TRN1 Q12 Q25 Q27 64 128 *) + 0x4edb6b2d; (* arm_TRN2 Q13 Q25 Q27 64 128 *) + 0x4edc2b4e; (* arm_TRN1 Q14 Q26 Q28 64 128 *) + 0x4edc6b4f; (* arm_TRN2 Q15 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b30; (* arm_TRN1 Q16 Q25 Q27 64 128 *) + 0x4edb6b31; (* arm_TRN2 Q17 Q25 Q27 64 128 *) + 0x4edc2b52; (* arm_TRN1 Q18 Q26 Q28 64 128 *) + 0x4edc6b53; (* arm_TRN2 Q19 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b34; (* arm_TRN1 Q20 Q25 Q27 64 128 *) + 0x4edb6b35; (* arm_TRN2 Q21 Q25 Q27 64 128 *) + 0x4edc2b56; (* arm_TRN1 Q22 Q26 Q28 64 128 *) + 0x4edc6b57; (* arm_TRN2 Q23 Q26 Q28 64 128 *) + 0xfd400019; (* arm_LDR D25 X0 (Immediate_Offset (word 0)) *) + 0xfd40009b; (* arm_LDR D27 X4 (Immediate_Offset (word 0)) *) + 0x4edb2b38; (* arm_TRN1 Q24 Q25 Q27 64 128 *) + 0xd1030000; (* arm_SUB X0 X0 (rvalue (word 192)) *) + 0x91064000; (* arm_ADD X0 X0 (rvalue (word 400)) *) + 0xa9401801; (* arm_LDP X1 X6 X0 (Immediate_Offset (iword (&0))) *) + 0xa941400b; (* arm_LDP X11 X16 X0 (Immediate_Offset (iword (&16))) *) + 0xa9420815; (* arm_LDP X21 X2 X0 (Immediate_Offset (iword (&32))) *) + 0xa9433007; (* arm_LDP X7 X12 X0 (Immediate_Offset (iword (&48))) *) + 0xa9445811; (* arm_LDP X17 X22 X0 (Immediate_Offset (iword (&64))) *) + 0xa9452003; (* arm_LDP X3 X8 X0 (Immediate_Offset (iword (&80))) *) + 0xa946700d; (* arm_LDP X13 X28 X0 (Immediate_Offset (iword (&96))) *) + 0xa9471017; (* arm_LDP X23 X4 X0 (Immediate_Offset (iword (&112))) *) + 0xa9483809; (* arm_LDP X9 X14 X0 (Immediate_Offset (iword (&128))) *) + 0xa9496013; (* arm_LDP X19 X24 X0 (Immediate_Offset (iword (&144))) *) + 0xa94a2805; (* arm_LDP X5 X10 X0 (Immediate_Offset (iword (&160))) *) + 0xa94b500f; (* arm_LDP X15 X20 X0 (Immediate_Offset (iword (&176))) *) + 0xf9406019; (* arm_LDR X25 X0 (Immediate_Offset (word 192)) *) + 0xd1064000; (* arm_SUB X0 X0 (rvalue (word 400)) *) + 0xca19031e; (* arm_EOR X30 X24 X25 *) + 0xca0a013b; (* arm_EOR X27 X9 X10 *) + 0xce05281e; (* arm_EOR3 Q30 Q0 Q5 Q10 *) + 0xce0f53de; (* arm_EOR3 Q30 Q30 Q15 Q20 *) + 0xca1503c0; (* arm_EOR X0 X30 X21 *) + 0xca06037a; (* arm_EOR X26 X27 X6 *) + 0xce062c3d; (* arm_EOR3 Q29 Q1 Q6 Q11 *) + 0xca07035b; (* arm_EOR X27 X26 X7 *) + 0xce1057bd; (* arm_EOR3 Q29 Q29 Q16 Q21 *) + 0xca16001d; (* arm_EOR X29 X0 X22 *) + 0xca1703ba; (* arm_EOR X26 X29 X23 *) + 0xce07305c; (* arm_EOR3 Q28 Q2 Q7 Q12 *) + 0xca05009d; (* arm_EOR X29 X4 X5 *) + 0xce115b9c; (* arm_EOR3 Q28 Q28 Q17 Q22 *) + 0xca0103be; (* arm_EOR X30 X29 X1 *) + 0xca080360; (* arm_EOR X0 X27 X8 *) + 0xce08347b; (* arm_EOR3 Q27 Q3 Q8 Q13 *) + 0xca0203dd; (* arm_EOR X29 X30 X2 *) + 0xce125f7b; (* arm_EOR3 Q27 Q27 Q18 Q23 *) + 0xca14027e; (* arm_EOR X30 X19 X20 *) + 0xca1003de; (* arm_EOR X30 X30 X16 *) + 0xce09389a; (* arm_EOR3 Q26 Q4 Q9 Q14 *) + 0xcac0ff5b; (* arm_EOR X27 X26 (Shiftedreg X0 ROR 63) *) + 0xce13635a; (* arm_EOR3 Q26 Q26 Q19 Q24 *) + 0xca1b0084; (* arm_EOR X4 X4 X27 *) + 0xca1103de; (* arm_EOR X30 X30 X17 *) + 0xce7c8fd9; (* arm_RAX1 Q25 Q30 Q28 *) + 0xca1c03de; (* arm_EOR X30 X30 X28 *) + 0xca0303bd; (* arm_EOR X29 X29 X3 *) + 0xce7a8f9c; (* arm_RAX1 Q28 Q28 Q26 *) + 0xcadefc00; (* arm_EOR X0 X0 (Shiftedreg X30 ROR 63) *) + 0xce7d8f5a; (* arm_RAX1 Q26 Q26 Q29 *) + 0xcaddffde; (* arm_EOR X30 X30 (Shiftedreg X29 ROR 63) *) + 0xca1e02d6; (* arm_EOR X22 X22 X30 *) + 0xce7b8fbd; (* arm_RAX1 Q29 Q29 Q27 *) + 0xca1e02f7; (* arm_EOR X23 X23 X30 *) + 0xce7e8f7b; (* arm_RAX1 Q27 Q27 Q30 *) + 0xf9006bf7; (* arm_STR X23 SP (Immediate_Offset (word 208)) *) + 0xca0f01d7; (* arm_EOR X23 X14 X15 *) + 0x6e3a1c1e; (* arm_EOR_VEC Q30 Q0 Q26 128 *) + 0xca0001ce; (* arm_EOR X14 X14 X0 *) + 0xce9d0840; (* arm_XAR Q0 Q2 Q29 (word 2) *) + 0xca0b02f7; (* arm_EOR X23 X23 X11 *) + 0xca0001ef; (* arm_EOR X15 X15 X0 *) + 0xce9d5582; (* arm_XAR Q2 Q12 Q29 (word 21) *) + 0xca1b0021; (* arm_EOR X1 X1 X27 *) + 0xce9c9dac; (* arm_XAR Q12 Q13 Q28 (word 39) *) + 0xca0c02f7; (* arm_EOR X23 X23 X12 *) + 0xca0d02f7; (* arm_EOR X23 X23 X13 *) + 0xce9be26d; (* arm_XAR Q13 Q19 Q27 (word 56) *) + 0xca00016b; (* arm_EOR X11 X11 X0 *) + 0xcad7ffbd; (* arm_EOR X29 X29 (Shiftedreg X23 ROR 63) *) + 0xce9c22f3; (* arm_XAR Q19 Q23 Q28 (word 8) *) + 0xcadafef7; (* arm_EOR X23 X23 (Shiftedreg X26 ROR 63) *) + 0xce9a5df7; (* arm_XAR Q23 Q15 Q26 (word 23) *) + 0xca0001ba; (* arm_EOR X26 X13 X0 *) + 0xca17038d; (* arm_EOR X13 X28 X23 *) + 0xce99fc2f; (* arm_XAR Q15 Q1 Q25 (word 63) *) + 0xca1e031c; (* arm_EOR X28 X24 X30 *) + 0xce9c2501; (* arm_XAR Q1 Q8 Q28 (word 9) *) + 0xca170218; (* arm_EOR X24 X16 X23 *) + 0xca1e02b0; (* arm_EOR X16 X21 X30 *) + 0xce994e08; (* arm_XAR Q8 Q16 Q25 (word 19) *) + 0xca1e0335; (* arm_EOR X21 X25 X30 *) + 0xce9de8f0; (* arm_XAR Q16 Q7 Q29 (word 58) *) + 0xca17027e; (* arm_EOR X30 X19 X23 *) + 0xca170293; (* arm_EOR X19 X20 X23 *) + 0xce9af547; (* arm_XAR Q7 Q10 Q26 (word 61) *) + 0xca170234; (* arm_EOR X20 X17 X23 *) + 0xce9c906a; (* arm_XAR Q10 Q3 Q28 (word 36) *) + 0xca000191; (* arm_EOR X17 X12 X0 *) + 0xca1b0040; (* arm_EOR X0 X2 X27 *) + 0xce9cae43; (* arm_XAR Q3 Q18 Q28 (word 43) *) + 0xca1d00c2; (* arm_EOR X2 X6 X29 *) + 0xce9dc632; (* arm_XAR Q18 Q17 Q29 (word 49) *) + 0xca1d0106; (* arm_EOR X6 X8 X29 *) + 0x8aedbf88; (* arm_BIC X8 X28 (Shiftedreg X13 ROR 47) *) + 0xce99d971; (* arm_XAR Q17 Q11 Q25 (word 54) *) + 0xca1b006c; (* arm_EOR X12 X3 X27 *) + 0x8af14da3; (* arm_BIC X3 X13 (Shiftedreg X17 ROR 19) *) + 0xce9bb12b; (* arm_XAR Q11 Q9 Q27 (word 44) *) + 0xca1b00a5; (* arm_EOR X5 X5 X27 *) + 0xce9d0ec9; (* arm_XAR Q9 Q22 Q29 (word 3) *) + 0xf9406bfb; (* arm_LDR X27 SP (Immediate_Offset (word 208)) *) + 0x8ae21639; (* arm_BIC X25 X17 (Shiftedreg X2 ROR 5) *) + 0xce9b65d6; (* arm_XAR Q22 Q14 Q27 (word 25) *) + 0xca1d0129; (* arm_EOR X9 X9 X29 *) + 0xce9aba8e; (* arm_XAR Q14 Q20 Q26 (word 46) *) + 0xcac5d337; (* arm_EOR X23 X25 (Shiftedreg X5 ROR 52) *) + 0xcac26063; (* arm_EOR X3 X3 (Shiftedreg X2 ROR 24) *) + 0xce9b9494; (* arm_XAR Q20 Q4 Q27 (word 37) *) + 0xcad10908; (* arm_EOR X8 X8 (Shiftedreg X17 ROR 2) *) + 0xce9bcb04; (* arm_XAR Q4 Q24 Q27 (word 50) *) + 0xca1d0151; (* arm_EOR X17 X10 X29 *) + 0x8af6bd99; (* arm_BIC X25 X12 (Shiftedreg X22 ROR 47) *) + 0xce99fab8; (* arm_XAR Q24 Q21 Q25 (word 62) *) + 0xca1d00fd; (* arm_EOR X29 X7 X29 *) + 0xce9a70b5; (* arm_XAR Q21 Q5 Q26 (word 28) *) + 0x8afb088a; (* arm_BIC X10 X4 (Shiftedreg X27 ROR 2) *) + 0x8afc28a7; (* arm_BIC X7 X5 (Shiftedreg X28 ROR 10) *) + 0xce9950db; (* arm_XAR Q27 Q6 Q25 (word 20) *) + 0xcad4c94a; (* arm_EOR X10 X10 (Shiftedreg X20 ROR 50) *) + 0xcacde4ed; (* arm_EOR X13 X7 (Shiftedreg X13 ROR 57) *) + 0xce272d45; (* arm_BCAX Q5 Q10 Q7 Q11 *) + 0x8ae5bc47; (* arm_BIC X7 X2 (Shiftedreg X5 ROR 47) *) + 0xce281d66; (* arm_BCAX Q6 Q11 Q8 Q7 *) + 0xcad89f22; (* arm_EOR X2 X25 (Shiftedreg X24 ROR 39) *) + 0x8aebe699; (* arm_BIC X25 X20 (Shiftedreg X11 ROR 57) *) + 0xce2920e7; (* arm_BCAX Q7 Q7 Q9 Q8 *) + 0x8ae46625; (* arm_BIC X5 X17 (Shiftedreg X4 ROR 25) *) + 0xce2a2508; (* arm_BCAX Q8 Q8 Q10 Q9 *) + 0xcad1d739; (* arm_EOR X25 X25 (Shiftedreg X17 ROR 53) *) + 0x8af1f171; (* arm_BIC X17 X11 (Shiftedreg X17 ROR 60) *) + 0xce2b2929; (* arm_BCAX Q9 Q9 Q11 Q10 *) + 0xcadce4fc; (* arm_EOR X28 X7 (Shiftedreg X28 ROR 57) *) + 0xce2c41ea; (* arm_BCAX Q10 Q15 Q12 Q16 *) + 0x8aeca927; (* arm_BIC X7 X9 (Shiftedreg X12 ROR 42) *) + 0xcad664e7; (* arm_EOR X7 X7 (Shiftedreg X22 ROR 25) *) + 0xce2d320b; (* arm_BCAX Q11 Q16 Q13 Q12 *) + 0x8af8e2d6; (* arm_BIC X22 X22 (Shiftedreg X24 ROR 56) *) + 0xce2e358c; (* arm_BCAX Q12 Q12 Q14 Q13 *) + 0x8aef7f18; (* arm_BIC X24 X24 (Shiftedreg X15 ROR 31) *) + 0xcacf5ed6; (* arm_EOR X22 X22 (Shiftedreg X15 ROR 23) *) + 0xce2f39ad; (* arm_BCAX Q13 Q13 Q15 Q14 *) + 0x8af4c374; (* arm_BIC X20 X27 (Shiftedreg X20 ROR 48) *) + 0xce303dce; (* arm_BCAX Q14 Q14 Q16 Q15 *) + 0x8ae941ef; (* arm_BIC X15 X15 (Shiftedreg X9 ROR 16) *) + 0xcacce9ec; (* arm_EOR X12 X15 (Shiftedreg X12 ROR 58) *) + 0xce31568f; (* arm_BCAX Q15 Q20 Q17 Q21 *) + 0xcadb6caf; (* arm_EOR X15 X5 (Shiftedreg X27 ROR 27) *) + 0xcacba685; (* arm_EOR X5 X20 (Shiftedreg X11 ROR 41) *) + 0xce3246b0; (* arm_BCAX Q16 Q21 Q18 Q17 *) + 0xf94007eb; (* arm_LDR X11 SP (Immediate_Offset (word 8)) *) + 0xce334a31; (* arm_BCAX Q17 Q17 Q19 Q18 *) + 0xcac45634; (* arm_EOR X20 X17 (Shiftedreg X4 ROR 21) *) + 0xcac9bf11; (* arm_EOR X17 X24 (Shiftedreg X9 ROR 47) *) + 0xce344e52; (* arm_BCAX Q18 Q18 Q20 Q19 *) + 0xd2800038; (* arm_MOV X24 (rvalue (word 1)) *) + 0xce355273; (* arm_BCAX Q19 Q19 Q21 Q20 *) + 0x8af02409; (* arm_BIC X9 X0 (Shiftedreg X16 ROR 9) *) + 0xf9000ff8; (* arm_STR X24 SP (Immediate_Offset (word 24)) *) + 0xce360414; (* arm_BCAX Q20 Q0 Q22 Q1 *) + 0x8ae1b3b8; (* arm_BIC X24 X29 (Shiftedreg X1 ROR 44) *) + 0xce375835; (* arm_BCAX Q21 Q1 Q23 Q22 *) + 0x8af5c83b; (* arm_BIC X27 X1 (Shiftedreg X21 ROR 50) *) + 0x8afdff44; (* arm_BIC X4 X26 (Shiftedreg X29 ROR 63) *) + 0xce385ed6; (* arm_BCAX Q22 Q22 Q24 Q23 *) + 0xcac45421; (* arm_EOR X1 X1 (Shiftedreg X4 ROR 21) *) + 0xce2062f7; (* arm_BCAX Q23 Q23 Q0 Q24 *) + 0xf940016b; (* arm_LDR X11 X11 (Immediate_Offset (word 0)) *) + 0x8afee6a4; (* arm_BIC X4 X21 (Shiftedreg X30 ROR 57) *) + 0xce210318; (* arm_BCAX Q24 Q24 Q1 Q0 *) + 0xcad57b15; (* arm_EOR X21 X24 (Shiftedreg X21 ROR 30) *) + 0xcad3b138; (* arm_EOR X24 X9 (Shiftedreg X19 ROR 44) *) + 0xce226fc0; (* arm_BCAX Q0 Q30 Q2 Q27 *) + 0x8ae615c9; (* arm_BIC X9 X14 (Shiftedreg X6 ROR 5) *) + 0xce230b61; (* arm_BCAX Q1 Q27 Q3 Q2 *) + 0xcac0ad29; (* arm_EOR X9 X9 (Shiftedreg X0 ROR 43) *) + 0x8ae098c0; (* arm_BIC X0 X6 (Shiftedreg X0 ROR 38) *) + 0xce240c42; (* arm_BCAX Q2 Q2 Q4 Q3 *) + 0xca0b0021; (* arm_EOR X1 X1 X11 *) + 0xce3e1063; (* arm_BCAX Q3 Q3 Q30 Q4 *) + 0xcada8c8b; (* arm_EOR X11 X4 (Shiftedreg X26 ROR 35) *) + 0xcad0bc04; (* arm_EOR X4 X0 (Shiftedreg X16 ROR 47) *) + 0xce3b7884; (* arm_BCAX Q4 Q4 Q27 Q30 *) + 0x8af38e00; (* arm_BIC X0 X16 (Shiftedreg X19 ROR 35) *) + 0xcadeaf70; (* arm_EOR X16 X27 (Shiftedreg X30 ROR 43) *) + 0x8afaabdb; (* arm_BIC X27 X30 (Shiftedreg X26 ROR 42) *) + 0xf9400bfe; (* arm_LDR X30 SP (Immediate_Offset (word 16)) *) + 0x4ddfcfdc; (* arm_LD1R Q28 X30 (Postimmediate_Offset (word 8)) 64 128 *) + 0x8aeea67a; (* arm_BIC X26 X19 (Shiftedreg X14 ROR 41) *) + 0xf9000bfe; (* arm_STR X30 SP (Immediate_Offset (word 16)) *) + 0xcace3013; (* arm_EOR X19 X0 (Shiftedreg X14 ROR 12) *) + 0xcac6bb4e; (* arm_EOR X14 X26 (Shiftedreg X6 ROR 46) *) + 0x6e3c1c00; (* arm_EOR_VEC Q0 Q0 Q28 128 *) + 0xcadda766; (* arm_EOR X6 X27 (Shiftedreg X29 ROR 41) *) + 0xcacbd1e0; (* arm_EOR X0 X15 (Shiftedreg X11 ROR 52) *) + 0xcacdc000; (* arm_EOR X0 X0 (Shiftedreg X13 ROR 48) *) + 0xce05281e; (* arm_EOR3 Q30 Q0 Q5 Q10 *) + 0xce0f53de; (* arm_EOR3 Q30 Q30 Q15 Q20 *) + 0xcac9e51a; (* arm_EOR X26 X8 (Shiftedreg X9 ROR 57) *) + 0xcace281b; (* arm_EOR X27 X0 (Shiftedreg X14 ROR 10) *) + 0xce062c3d; (* arm_EOR3 Q29 Q1 Q6 Q11 *) + 0xcadcfe1d; (* arm_EOR X29 X16 (Shiftedreg X28 ROR 63) *) + 0xce1057bd; (* arm_EOR3 Q29 Q29 Q16 Q21 *) + 0xcac6cf5a; (* arm_EOR X26 X26 (Shiftedreg X6 ROR 51) *) + 0xcad6cafe; (* arm_EOR X30 X23 (Shiftedreg X22 ROR 50) *) + 0xce07305c; (* arm_EOR3 Q28 Q2 Q7 Q12 *) + 0xcaca7f40; (* arm_EOR X0 X26 (Shiftedreg X10 ROR 31) *) + 0xcad397bd; (* arm_EOR X29 X29 (Shiftedreg X19 ROR 37) *) + 0xce115b9c; (* arm_EOR3 Q28 Q28 Q17 Q22 *) + 0xcacc177b; (* arm_EOR X27 X27 (Shiftedreg X12 ROR 5) *) + 0xce08347b; (* arm_EOR3 Q27 Q3 Q8 Q13 *) + 0xcad88bde; (* arm_EOR X30 X30 (Shiftedreg X24 ROR 34) *) + 0xcac76c00; (* arm_EOR X0 X0 (Shiftedreg X7 ROR 27) *) + 0xce125f7b; (* arm_EOR3 Q27 Q27 Q18 Q23 *) + 0xcad56bda; (* arm_EOR X26 X30 (Shiftedreg X21 ROR 26) *) + 0xce09389a; (* arm_EOR3 Q26 Q4 Q9 Q14 *) + 0xcad93f5a; (* arm_EOR X26 X26 (Shiftedreg X25 ROR 15) *) + 0x93dbfb7e; (* arm_ROR X30 X27 62 *) + 0xce13635a; (* arm_EOR3 Q26 Q26 Q19 Q24 *) + 0xcadae7de; (* arm_EOR X30 X30 (Shiftedreg X26 ROR 57) *) + 0x93daeb5a; (* arm_ROR X26 X26 58 *) + 0xce7c8fd9; (* arm_RAX1 Q25 Q30 Q28 *) + 0xca1003d0; (* arm_EOR X16 X30 X16 *) + 0xce7a8f9c; (* arm_RAX1 Q28 Q28 Q26 *) + 0xcadcffdc; (* arm_EOR X28 X30 (Shiftedreg X28 ROR 63) *) + 0xf9006bfc; (* arm_STR X28 SP (Immediate_Offset (word 208)) *) + 0xce7d8f5a; (* arm_RAX1 Q26 Q26 Q29 *) + 0xcad193bd; (* arm_EOR X29 X29 (Shiftedreg X17 ROR 36) *) + 0xcac2f43c; (* arm_EOR X28 X1 (Shiftedreg X2 ROR 61) *) + 0xce7b8fbd; (* arm_RAX1 Q29 Q29 Q27 *) + 0xcad397d3; (* arm_EOR X19 X30 (Shiftedreg X19 ROR 37) *) + 0xce7e8f7b; (* arm_RAX1 Q27 Q27 Q30 *) + 0xcad40bbd; (* arm_EOR X29 X29 (Shiftedreg X20 ROR 2) *) + 0xcac4db9c; (* arm_EOR X28 X28 (Shiftedreg X4 ROR 54) *) + 0x6e3a1c1e; (* arm_EOR_VEC Q30 Q0 Q26 128 *) + 0xcac0df5a; (* arm_EOR X26 X26 (Shiftedreg X0 ROR 55) *) + 0xce9d0840; (* arm_XAR Q0 Q2 Q29 (word 2) *) + 0xcac39f9c; (* arm_EOR X28 X28 (Shiftedreg X3 ROR 39) *) + 0xcac5679c; (* arm_EOR X28 X28 (Shiftedreg X5 ROR 25) *) + 0xce9d5582; (* arm_XAR Q2 Q12 Q29 (word 21) *) + 0x93c0e000; (* arm_ROR X0 X0 56 *) + 0xcaddfc00; (* arm_EOR X0 X0 (Shiftedreg X29 ROR 63) *) + 0xce9c9dac; (* arm_XAR Q12 Q13 Q28 (word 39) *) + 0xcadbf79b; (* arm_EOR X27 X28 (Shiftedreg X27 ROR 61) *) + 0xce9be26d; (* arm_XAR Q13 Q19 Q27 (word 56) *) + 0xcacdb80d; (* arm_EOR X13 X0 (Shiftedreg X13 ROR 46) *) + 0xcadcffbc; (* arm_EOR X28 X29 (Shiftedreg X28 ROR 63) *) + 0xce9c22f3; (* arm_XAR Q19 Q23 Q28 (word 8) *) + 0xcad40bdd; (* arm_EOR X29 X30 (Shiftedreg X20 ROR 2) *) + 0xcac39f54; (* arm_EOR X20 X26 (Shiftedreg X3 ROR 39) *) + 0xce9a5df7; (* arm_XAR Q23 Q15 Q26 (word 23) *) + 0xcacbc80b; (* arm_EOR X11 X0 (Shiftedreg X11 ROR 50) *) + 0xce99fc2f; (* arm_XAR Q15 Q1 Q25 (word 63) *) + 0xcad92799; (* arm_EOR X25 X28 (Shiftedreg X25 ROR 9) *) + 0xcad55383; (* arm_EOR X3 X28 (Shiftedreg X21 ROR 20) *) + 0xce9c2501; (* arm_XAR Q1 Q8 Q28 (word 9) *) + 0xca010355; (* arm_EOR X21 X26 X1 *) + 0xce994e08; (* arm_XAR Q8 Q16 Q25 (word 19) *) + 0xcac9c769; (* arm_EOR X9 X27 (Shiftedreg X9 ROR 49) *) + 0xcad87398; (* arm_EOR X24 X28 (Shiftedreg X24 ROR 28) *) + 0xce9de8f0; (* arm_XAR Q16 Q7 Q29 (word 58) *) + 0xcad193c1; (* arm_EOR X1 X30 (Shiftedreg X17 ROR 36) *) + 0xcace200e; (* arm_EOR X14 X0 (Shiftedreg X14 ROR 8) *) + 0xce9af547; (* arm_XAR Q7 Q10 Q26 (word 61) *) + 0xcad6b396; (* arm_EOR X22 X28 (Shiftedreg X22 ROR 44) *) + 0xce9c906a; (* arm_XAR Q10 Q3 Q28 (word 36) *) + 0xcac8e368; (* arm_EOR X8 X27 (Shiftedreg X8 ROR 56) *) + 0xcac74f71; (* arm_EOR X17 X27 (Shiftedreg X7 ROR 19) *) + 0xce9cae43; (* arm_XAR Q3 Q18 Q28 (word 43) *) + 0xcacff80f; (* arm_EOR X15 X0 (Shiftedreg X15 ROR 62) *) + 0x8af6be87; (* arm_BIC X7 X20 (Shiftedreg X22 ROR 47) *) + 0xce9dc632; (* arm_XAR Q18 Q17 Q29 (word 49) *) + 0xcac4db44; (* arm_EOR X4 X26 (Shiftedreg X4 ROR 54) *) + 0xce99d971; (* arm_XAR Q17 Q11 Q25 (word 54) *) + 0xcacc0c00; (* arm_EOR X0 X0 (Shiftedreg X12 ROR 3) *) + 0xcad7eb9c; (* arm_EOR X28 X28 (Shiftedreg X23 ROR 58) *) + 0xce9bb12b; (* arm_XAR Q11 Q9 Q27 (word 44) *) + 0xcac2f757; (* arm_EOR X23 X26 (Shiftedreg X2 ROR 61) *) + 0xce9d0ec9; (* arm_XAR Q9 Q22 Q29 (word 3) *) + 0xcac5675a; (* arm_EOR X26 X26 (Shiftedreg X5 ROR 25) *) + 0xcad09ce2; (* arm_EOR X2 X7 (Shiftedreg X16 ROR 39) *) + 0xce9b65d6; (* arm_XAR Q22 Q14 Q27 (word 25) *) + 0x8af4a927; (* arm_BIC X7 X9 (Shiftedreg X20 ROR 42) *) + 0x8ae941fe; (* arm_BIC X30 X15 (Shiftedreg X9 ROR 16) *) + 0xce9aba8e; (* arm_XAR Q14 Q20 Q26 (word 46) *) + 0xcad664e7; (* arm_EOR X7 X7 (Shiftedreg X22 ROR 25) *) + 0xce9b9494; (* arm_XAR Q20 Q4 Q27 (word 37) *) + 0xcad4ebcc; (* arm_EOR X12 X30 (Shiftedreg X20 ROR 58) *) + 0x8af0e2d4; (* arm_BIC X20 X22 (Shiftedreg X16 ROR 56) *) + 0xce9bcb04; (* arm_XAR Q4 Q24 Q27 (word 50) *) + 0xcac6af7e; (* arm_EOR X30 X27 (Shiftedreg X6 ROR 43) *) + 0xcacf5e96; (* arm_EOR X22 X20 (Shiftedreg X15 ROR 23) *) + 0xce99fab8; (* arm_XAR Q24 Q21 Q25 (word 62) *) + 0x8aedaa66; (* arm_BIC X6 X19 (Shiftedreg X13 ROR 42) *) + 0xce9a70b5; (* arm_XAR Q21 Q5 Q26 (word 28) *) + 0xcad1a4c6; (* arm_EOR X6 X6 (Shiftedreg X17 ROR 41) *) + 0x8af1fda5; (* arm_BIC X5 X13 (Shiftedreg X17 ROR 63) *) + 0xce9950db; (* arm_XAR Q27 Q6 Q25 (word 20) *) + 0xcac556a5; (* arm_EOR X5 X21 (Shiftedreg X5 ROR 21) *) + 0xce272d45; (* arm_BCAX Q5 Q10 Q7 Q11 *) + 0x8af5b231; (* arm_BIC X17 X17 (Shiftedreg X21 ROR 44) *) + 0xcaca5f7b; (* arm_EOR X27 X27 (Shiftedreg X10 ROR 23) *) + 0xce281d66; (* arm_BCAX Q6 Q11 Q8 Q7 *) + 0x8af9cab5; (* arm_BIC X21 X21 (Shiftedreg X25 ROR 50) *) + 0x8ae46774; (* arm_BIC X20 X27 (Shiftedreg X4 ROR 25) *) + 0xce2920e7; (* arm_BCAX Q7 Q7 Q9 Q8 *) + 0x8aef7e0a; (* arm_BIC X10 X16 (Shiftedreg X15 ROR 31) *) + 0xce2a2508; (* arm_BCAX Q8 Q8 Q10 Q9 *) + 0xcad3aeb0; (* arm_EOR X16 X21 (Shiftedreg X19 ROR 43) *) + 0xcad97a35; (* arm_EOR X21 X17 (Shiftedreg X25 ROR 30) *) + 0xce2b2929; (* arm_BCAX Q9 Q9 Q11 Q10 *) + 0x8af3e733; (* arm_BIC X19 X25 (Shiftedreg X19 ROR 57) *) + 0xf9400ff9; (* arm_LDR X25 SP (Immediate_Offset (word 24)) *) + 0xce2c41ea; (* arm_BCAX Q10 Q15 Q12 Q16 *) + 0xcac9bd51; (* arm_EOR X17 X10 (Shiftedreg X9 ROR 47) *) + 0xce2d320b; (* arm_BCAX Q11 Q16 Q13 Q12 *) + 0xf94007e9; (* arm_LDR X9 SP (Immediate_Offset (word 8)) *) + 0xcadc6e8f; (* arm_EOR X15 X20 (Shiftedreg X28 ROR 27) *) + 0xce2e358c; (* arm_BCAX Q12 Q12 Q14 Q13 *) + 0x8afc0894; (* arm_BIC X20 X4 (Shiftedreg X28 ROR 2) *) + 0xce2f39ad; (* arm_BCAX Q13 Q13 Q15 Q14 *) + 0xcac1ca8a; (* arm_EOR X10 X20 (Shiftedreg X1 ROR 50) *) + 0x8afbf174; (* arm_BIC X20 X11 (Shiftedreg X27 ROR 60) *) + 0xce303dce; (* arm_BCAX Q14 Q14 Q16 Q15 *) + 0xcac45694; (* arm_EOR X20 X20 (Shiftedreg X4 ROR 21) *) + 0x8ae1c384; (* arm_BIC X4 X28 (Shiftedreg X1 ROR 48) *) + 0xce31568f; (* arm_BCAX Q15 Q20 Q17 Q21 *) + 0x8aebe421; (* arm_BIC X1 X1 (Shiftedreg X11 ROR 57) *) + 0xce3246b0; (* arm_BCAX Q16 Q21 Q18 Q17 *) + 0xf879793c; (* arm_LDR X28 X9 (Shiftreg_Offset X25 3) *) + 0xf9406be9; (* arm_LDR X9 SP (Immediate_Offset (word 208)) *) + 0xce334a31; (* arm_BCAX Q17 Q17 Q19 Q18 *) + 0x91000739; (* arm_ADD X25 X25 (rvalue (word 1)) *) + 0xf9000ff9; (* arm_STR X25 SP (Immediate_Offset (word 24)) *) + 0xce344e52; (* arm_BCAX Q18 Q18 Q20 Q19 *) + 0xf1005f3f; (* arm_CMP X25 (rvalue (word 23)) *) + 0xce355273; (* arm_BCAX Q19 Q19 Q21 Q20 *) + 0xcadbd439; (* arm_EOR X25 X1 (Shiftedreg X27 ROR 53) *) + 0x8afabfdb; (* arm_BIC X27 X30 (Shiftedreg X26 ROR 47) *) + 0xce360414; (* arm_BCAX Q20 Q0 Q22 Q1 *) + 0xca1c00a1; (* arm_EOR X1 X5 X28 *) + 0xce375835; (* arm_BCAX Q21 Q1 Q23 Q22 *) + 0xcacba485; (* arm_EOR X5 X4 (Shiftedreg X11 ROR 41) *) + 0xcacd8e6b; (* arm_EOR X11 X19 (Shiftedreg X13 ROR 35) *) + 0xce385ed6; (* arm_BCAX Q22 Q22 Q24 Q23 *) + 0x8af82b4d; (* arm_BIC X13 X26 (Shiftedreg X24 ROR 10) *) + 0xcad8e77c; (* arm_EOR X28 X27 (Shiftedreg X24 ROR 57) *) + 0xce2062f7; (* arm_BCAX Q23 Q23 Q0 Q24 *) + 0x8ae9bf1b; (* arm_BIC X27 X24 (Shiftedreg X9 ROR 47) *) + 0xce210318; (* arm_BCAX Q24 Q24 Q1 Q0 *) + 0x8ae326f3; (* arm_BIC X19 X23 (Shiftedreg X3 ROR 9) *) + 0x8aeea7a4; (* arm_BIC X4 X29 (Shiftedreg X14 ROR 41) *) + 0xce226fc0; (* arm_BCAX Q0 Q30 Q2 Q27 *) + 0xcaddb278; (* arm_EOR X24 X19 (Shiftedreg X29 ROR 44) *) + 0x8afd8c7d; (* arm_BIC X29 X3 (Shiftedreg X29 ROR 35) *) + 0xce230b61; (* arm_BCAX Q1 Q27 Q3 Q2 *) + 0xcac9e5ad; (* arm_EOR X13 X13 (Shiftedreg X9 ROR 57) *) + 0xce240c42; (* arm_BCAX Q2 Q2 Q4 Q3 *) + 0xcace33b3; (* arm_EOR X19 X29 (Shiftedreg X14 ROR 12) *) + 0x8ae04d3d; (* arm_BIC X29 X9 (Shiftedreg X0 ROR 19) *) + 0xce3e1063; (* arm_BCAX Q3 Q3 Q30 Q4 *) + 0x8ae815ce; (* arm_BIC X14 X14 (Shiftedreg X8 ROR 5) *) + 0xce3b7884; (* arm_BCAX Q4 Q4 Q27 Q30 *) + 0xcad7adc9; (* arm_EOR X9 X14 (Shiftedreg X23 ROR 43) *) + 0xcac8b88e; (* arm_EOR X14 X4 (Shiftedreg X8 ROR 46) *) + 0x8af79917; (* arm_BIC X23 X8 (Shiftedreg X23 ROR 38) *) + 0xcac00b68; (* arm_EOR X8 X27 (Shiftedreg X0 ROR 2) *) + 0xcac3bee4; (* arm_EOR X4 X23 (Shiftedreg X3 ROR 47) *) + 0x8afe1403; (* arm_BIC X3 X0 (Shiftedreg X30 ROR 5) *) + 0xcadad077; (* arm_EOR X23 X3 (Shiftedreg X26 ROR 52) *) + 0xcade63a3; (* arm_EOR X3 X29 (Shiftedreg X30 ROR 24) *) + 0xf9400bfe; (* arm_LDR X30 SP (Immediate_Offset (word 16)) *) + 0x4ddfcfdc; (* arm_LD1R Q28 X30 (Postimmediate_Offset (word 8)) 64 128 *) + 0xf9000bfe; (* arm_STR X30 SP (Immediate_Offset (word 16)) *) + 0x6e3c1c00; (* arm_EOR_VEC Q0 Q0 Q28 128 *) + 0x54ffe96d; (* arm_BLE (word 2096428) *) + 0x93c2f442; (* arm_ROR X2 X2 61 *) + 0x93c39c63; (* arm_ROR X3 X3 39 *) + 0x93c4d884; (* arm_ROR X4 X4 54 *) + 0x93c564a5; (* arm_ROR X5 X5 25 *) + 0x93c6acc6; (* arm_ROR X6 X6 43 *) + 0x93c74ce7; (* arm_ROR X7 X7 19 *) + 0x93c8e108; (* arm_ROR X8 X8 56 *) + 0x93c9c529; (* arm_ROR X9 X9 49 *) + 0x93ca5d4a; (* arm_ROR X10 X10 23 *) + 0x93cbc96b; (* arm_ROR X11 X11 50 *) + 0x93cc0d8c; (* arm_ROR X12 X12 3 *) + 0x93cdb9ad; (* arm_ROR X13 X13 46 *) + 0x93ce21ce; (* arm_ROR X14 X14 8 *) + 0x93cff9ef; (* arm_ROR X15 X15 62 *) + 0x93d19231; (* arm_ROR X17 X17 36 *) + 0x93dcff9c; (* arm_ROR X28 X28 63 *) + 0x93d39673; (* arm_ROR X19 X19 37 *) + 0x93d40a94; (* arm_ROR X20 X20 2 *) + 0x93d552b5; (* arm_ROR X21 X21 20 *) + 0x93d6b2d6; (* arm_ROR X22 X22 44 *) + 0x93d7eaf7; (* arm_ROR X23 X23 58 *) + 0x93d87318; (* arm_ROR X24 X24 28 *) + 0x93d92739; (* arm_ROR X25 X25 9 *) + 0xf94003e0; (* arm_LDR X0 SP (Immediate_Offset (word 0)) *) + 0x91064000; (* arm_ADD X0 X0 (rvalue (word 400)) *) + 0xa9001801; (* arm_STP X1 X6 X0 (Immediate_Offset (iword (&0))) *) + 0xa901400b; (* arm_STP X11 X16 X0 (Immediate_Offset (iword (&16))) *) + 0xa9020815; (* arm_STP X21 X2 X0 (Immediate_Offset (iword (&32))) *) + 0xa9033007; (* arm_STP X7 X12 X0 (Immediate_Offset (iword (&48))) *) + 0xa9045811; (* arm_STP X17 X22 X0 (Immediate_Offset (iword (&64))) *) + 0xa9052003; (* arm_STP X3 X8 X0 (Immediate_Offset (iword (&80))) *) + 0xa906700d; (* arm_STP X13 X28 X0 (Immediate_Offset (iword (&96))) *) + 0xa9071017; (* arm_STP X23 X4 X0 (Immediate_Offset (iword (&112))) *) + 0xa9083809; (* arm_STP X9 X14 X0 (Immediate_Offset (iword (&128))) *) + 0xa9096013; (* arm_STP X19 X24 X0 (Immediate_Offset (iword (&144))) *) + 0xa90a2805; (* arm_STP X5 X10 X0 (Immediate_Offset (iword (&160))) *) + 0xa90b500f; (* arm_STP X15 X20 X0 (Immediate_Offset (iword (&176))) *) + 0xf9006019; (* arm_STR X25 X0 (Immediate_Offset (word 192)) *) + 0xd1064000; (* arm_SUB X0 X0 (rvalue (word 400)) *) + 0x91032004; (* arm_ADD X4 X0 (rvalue (word 200)) *) + 0x4ec12819; (* arm_TRN1 Q25 Q0 Q1 64 128 *) + 0x4ec3285a; (* arm_TRN1 Q26 Q2 Q3 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec1681b; (* arm_TRN2 Q27 Q0 Q1 64 128 *) + 0x4ec3685c; (* arm_TRN2 Q28 Q2 Q3 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ec52899; (* arm_TRN1 Q25 Q4 Q5 64 128 *) + 0x4ec728da; (* arm_TRN1 Q26 Q6 Q7 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec5689b; (* arm_TRN2 Q27 Q4 Q5 64 128 *) + 0x4ec768dc; (* arm_TRN2 Q28 Q6 Q7 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ec92919; (* arm_TRN1 Q25 Q8 Q9 64 128 *) + 0x4ecb295a; (* arm_TRN1 Q26 Q10 Q11 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec9691b; (* arm_TRN2 Q27 Q8 Q9 64 128 *) + 0x4ecb695c; (* arm_TRN2 Q28 Q10 Q11 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ecd2999; (* arm_TRN1 Q25 Q12 Q13 64 128 *) + 0x4ecf29da; (* arm_TRN1 Q26 Q14 Q15 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ecd699b; (* arm_TRN2 Q27 Q12 Q13 64 128 *) + 0x4ecf69dc; (* arm_TRN2 Q28 Q14 Q15 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ed12a19; (* arm_TRN1 Q25 Q16 Q17 64 128 *) + 0x4ed32a5a; (* arm_TRN1 Q26 Q18 Q19 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ed16a1b; (* arm_TRN2 Q27 Q16 Q17 64 128 *) + 0x4ed36a5c; (* arm_TRN2 Q28 Q18 Q19 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ed52a99; (* arm_TRN1 Q25 Q20 Q21 64 128 *) + 0x4ed72ada; (* arm_TRN1 Q26 Q22 Q23 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ed56a9b; (* arm_TRN2 Q27 Q20 Q21 64 128 *) + 0x4ed76adc; (* arm_TRN2 Q28 Q22 Q23 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0xfd000018; (* arm_STR D24 X0 (Immediate_Offset (word 0)) *) + 0x4ed86b19; (* arm_TRN2 Q25 Q24 Q24 64 128 *) + 0xfd000099; (* arm_STR D25 X4 (Immediate_Offset (word 0)) *) + 0x6d4927e8; (* arm_LDP D8 D9 SP (Immediate_Offset (iword (&144))) *) + 0x6d4a2fea; (* arm_LDP D10 D11 SP (Immediate_Offset (iword (&160))) *) + 0x6d4b37ec; (* arm_LDP D12 D13 SP (Immediate_Offset (iword (&176))) *) + 0x6d4c3fee; (* arm_LDP D14 D15 SP (Immediate_Offset (iword (&192))) *) + 0xa94353f3; (* arm_LDP X19 X20 SP (Immediate_Offset (iword (&48))) *) + 0xa9445bf5; (* arm_LDP X21 X22 SP (Immediate_Offset (iword (&64))) *) + 0xa94563f7; (* arm_LDP X23 X24 SP (Immediate_Offset (iword (&80))) *) + 0xa9466bf9; (* arm_LDP X25 X26 SP (Immediate_Offset (iword (&96))) *) + 0xa94773fb; (* arm_LDP X27 X28 SP (Immediate_Offset (iword (&112))) *) + 0xa9487bfd; (* arm_LDP X29 X30 SP (Immediate_Offset (iword (&128))) *) + 0x910383ff; (* arm_ADD SP SP (rvalue (word 224)) *) + 0xd65f03c0 (* arm_RET X30 *) +];; +(*** BYTECODE END ***) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC = ARM_MK_EXEC_RULE keccak_f1600_x3_v84a_scalar_mc;; + +(*** Additional lazy/deferred rotations in the implementation, row-major ***) + +let deferred_rotates = define + `deferred_rotates = + [ 0; 21; 14; 0; 44; + 3; 45; 61; 28; 20; + 25; 8; 18; 1; 6; + 10; 15; 56; 27; 36; + 39; 41; 2; 62; 55]`;; + +(* ------------------------------------------------------------------------- *) +(* Code length constants *) +(* ------------------------------------------------------------------------- *) + +let LENGTH_KECCAK_F1600_X3_V84A_SCALAR_HYBRID_MC = + REWRITE_CONV[keccak_f1600_x3_v84a_scalar_mc] `LENGTH keccak_f1600_x3_v84a_scalar_mc` + |> CONV_RULE (RAND_CONV LENGTH_CONV);; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH = 44`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH = 48`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START = KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END = LENGTH keccak_f1600_x3_v84a_scalar_mc - KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH`;; + +let LENGTH_SIMPLIFY_CONV = + REWRITE_CONV[LENGTH_KECCAK_F1600_X3_V84A_SCALAR_HYBRID_MC; + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START; KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END; + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH; KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH] THENC + NUM_REDUCE_CONV THENC REWRITE_CONV [ADD_0];; + +(* ------------------------------------------------------------------------- *) +(* Correctness proof *) +(* ------------------------------------------------------------------------- *) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORRECT = prove + (`!a rc A1 A2 A3 pc stackpointer. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (stackpointer,216) /\ + ALLPAIRS nonoverlapping + [(a,600); (stackpointer,216)] + [(word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc); (rc,192)] + ==> ensures arm + (\s. aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word (pc + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START) /\ + read SP s = stackpointer /\ + C_ARGUMENTS [a; rc] s /\ + wordlist_from_memory(a,25) s = A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = A3 /\ + wordlist_from_memory(rc,24) s = round_constants) + (\s. read PC s = word(pc + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END) /\ + wordlist_from_memory(a,25) s = keccak 24 A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = + keccak 24 A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = + keccak 24 A3) + (MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI ,, + MAYCHANGE [X19; X20; X21; X22; X23; X24; X25; X26; + X27; X28; X29; X30] ,, + MAYCHANGE [Q8; Q9; Q10; Q11; Q12; Q13; Q14; Q15] ,, + MAYCHANGE [memory :> bytes(a,600); + memory :> bytes(stackpointer,32); + memory :> bytes(word_add stackpointer (word 208),8)])`, + CONV_TAC LENGTH_SIMPLIFY_CONV THEN + MAP_EVERY X_GEN_TAC + [`a:int64`; `rc:int64`; `A1:int64 list`; `A2:int64 list`; + `A3:int64 list`; `pc:num`; `stackpointer:int64`] THEN + REWRITE_TAC[fst KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC] THEN + REWRITE_TAC[MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI; C_ARGUMENTS; + ALL; ALLPAIRS; NONOVERLAPPING_CLAUSES] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN ASSUME_TAC) THEN + + (*** Establish once and for all that all the Ai have length 25 ***) + + ASM_CASES_TAC + `LENGTH(A1:int64 list) = 25 /\ LENGTH(A2:int64 list) = 25 /\ + LENGTH(A3:int64 list) = 25` + THENL + [FIRST_X_ASSUM(CONJUNCTS_THEN STRIP_ASSUME_TAC); + ENSURES_INIT_TAC "s0" THEN MATCH_MP_TAC(TAUT `F ==> p`) THEN + REPEAT(FIRST_X_ASSUM(MP_TAC o AP_TERM `LENGTH:int64 list->num`)) THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + REWRITE_TAC[LENGTH; ARITH] THEN ASM_MESON_TAC[]] THEN + + (*** The main loop: one scalar and one vector round per iteration ***) + + ENSURES_WHILE_PAUP_TAC `1` `24` `pc + 0x3dc` `pc + 0x6b0` + `\i s. + (read SP s = stackpointer /\ + wordlist_from_memory(rc,24) s:int64 list = round_constants /\ + read (memory :> bytes64 stackpointer) s = a /\ + read (memory :> bytes64 (word_add stackpointer (word 8))) s = rc /\ + read (memory :> bytes64 (word_add stackpointer (word 16))) s = + word_add rc (word(8 * i)) /\ + read (memory :> bytes64 (word_add stackpointer (word 24))) s = + word i /\ + [read Q0 s; read Q1 s; read Q2 s; read Q3 s; read Q4 s; + read Q5 s; read Q6 s; read Q7 s; read Q8 s; read Q9 s; + read Q10 s; read Q11 s; read Q12 s; read Q13 s; read Q14 s; + read Q15 s; read Q16 s; read Q17 s; read Q18 s; read Q19 s; + read Q20 s; read Q21 s; read Q22 s; read Q23 s; read Q24 s] = + MAP2 word_join (keccak i A2) (keccak i A1) /\ + [read X1 s; read X6 s; read X11 s; read X16 s; read X21 s; + read X2 s; read X7 s; read X12 s; read X17 s; read X22 s; + read X3 s; read X8 s; read X13 s; read X28 s; read X23 s; + read X4 s; read X9 s; read X14 s; read X19 s; read X24 s; + read X5 s; read X10 s; read X15 s; read X20 s; read X25 s] = + MAP2 word_rol (keccak i A3) (MAP (\j. 64 - j) deferred_rotates)) /\ + (condition_semantics Condition_LE s <=> i < 24)` THEN + REWRITE_TAC[condition_semantics] THEN REPEAT CONJ_TAC THENL + [ARITH_TAC; + + (*** Initial holding of the invariant ***) + + REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC; + WORDLIST_FROM_MEMORY_CONV `wordlist_from_memory(rc,24) s:int64 list`] THEN + ENSURES_INIT_TAC "s0" THEN + BIGNUM_DIGITIZE_TAC "A_" `read (memory :> bytes (a,8 * 75)) s0` THEN + REPEAT(FIRST_X_ASSUM + (MP_TAC o CONV_RULE(LAND_CONV WORDLIST_FROM_MEMORY_CONV))) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[] THEN REPEAT DISCH_TAC THEN + MEMORY_128_FROM_64_TAC "a" 0 12 THEN + MEMORY_128_FROM_64_TAC "a" 200 12 THEN + ASM_REWRITE_TAC[WORD_ADD_0] THEN REPEAT STRIP_TAC THEN + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--236) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + CONJ_TAC THENL [CONV_TAC WORD_RULE; ALL_TAC] THEN + CONJ_TAC THENL + [GEN_REWRITE_TAC (RAND_CONV o ONCE_DEPTH_CONV) + [ARITH_RULE `1 = 0 + 1`] THEN + REWRITE_TAC[keccak; round_constants; keccak_round] THEN + MAP_EVERY EXPAND_TAC ["A1"; "A2"; "A3"] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC; + + GEN_REWRITE_TAC (RAND_CONV o ONCE_DEPTH_CONV) + [ARITH_RULE `1 = 0 + 1`] THEN + REWRITE_TAC[keccak; round_constants; keccak_round; + deferred_rotates; MAP] THEN + CONV_TAC(RAND_CONV NUM_REDUCE_CONV) THEN + MAP_EVERY EXPAND_TAC ["A1"; "A2"; "A3"] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC]; + + (*** Preservation of the invariant including end condition code ***) + + X_GEN_TAC `i:num` THEN STRIP_TAC THEN + VAL_INT64_TAC `i:num` THEN + MP_TAC(WORD_RULE `word_add (word i) (word 1):int64 = word(i + 1)`) THEN + DISCH_TAC THEN + CONV_TAC(RATOR_CONV(LAND_CONV + (ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC] THEN + MP_TAC(ISPECL [`A1:int64 list`; `i:num`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A2:int64 list`; `i:num`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A3:int64 list`; `i:num`] LENGTH_KECCAK) THEN + ASM_REWRITE_TAC[IMP_IMP] THEN REWRITE_TAC[LENGTH_EQ_25] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN SUBST1_TAC) THEN + REWRITE_TAC[deferred_rotates; MAP] THEN CONV_TAC NUM_REDUCE_CONV THEN + REWRITE_TAC[MAP2; CONS_11; GSYM CONJ_ASSOC] THEN + ENSURES_INIT_TAC "s0" THEN + + (*** Both rc cursors read rc[i]: [sp+16] for the vector round and ***) + (*** [sp+8] indexed by the counter [sp+24] for the scalar round ***) + + SUBGOAL_THEN + `read (memory :> bytes64 (word_add rc (word(8 * i)))) s0 = + EL i round_constants` + ASSUME_TAC THENL + [UNDISCH_TAC `i < 24` THEN SPEC_TAC(`i:num`,`i:num`) THEN + CONV_TAC EXPAND_CASES_CONV THEN + CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV) THEN + ASM_REWRITE_TAC[round_constants; WORD_ADD_0] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN REWRITE_TAC[]; + ALL_TAC] THEN + + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--181) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + REWRITE_TAC[CONJ_ASSOC] THEN CONJ_TAC THENL + [REWRITE_TAC[GSYM CONJ_ASSOC]; + UNDISCH_TAC `i < 24` THEN SPEC_TAC(`i:num`,`i:num`) THEN + CONV_TAC EXPAND_CASES_CONV THEN + CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV)] THEN + CONJ_TAC THENL + [CONV_TAC(LAND_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[]; + ALL_TAC] THEN + CONJ_TAC THENL [CONV_TAC WORD_RULE; ALL_TAC] THEN + + (*** One round for both the vector and the scalar lanes ***) + + CONJ_TAC THEN + REWRITE_TAC[keccak; keccak_round] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC; + + (*** The trivial loop-back goal ***) + + X_GEN_TAC `i:num` THEN STRIP_TAC THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ARM_SIM_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC [1] THEN + VAL_INT64_TAC `i:num` THEN + ASM_REWRITE_TAC[] THEN CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV); + + ALL_TAC] THEN + + (*** The finale making final deferred rotations and writing back ***) + + CONV_TAC NUM_REDUCE_CONV THEN + MP_TAC(ISPECL [`A1:int64 list`; `24`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A2:int64 list`; `24`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A3:int64 list`; `24`] LENGTH_KECCAK) THEN + ASM_REWRITE_TAC[] THEN + REWRITE_TAC[IMP_IMP] THEN REWRITE_TAC[LENGTH_EQ_25] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN SUBST1_TAC) THEN + REWRITE_TAC[deferred_rotates; MAP] THEN CONV_TAC NUM_REDUCE_CONV THEN + REWRITE_TAC[MAP2; CONS_11; GSYM CONJ_ASSOC] THEN + CONV_TAC(RATOR_CONV(LAND_CONV + (ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC] THEN + REWRITE_TAC[GSYM round_constants] THEN + ENSURES_INIT_TAC "s0" THEN + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--80) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[] THEN + RULE_ASSUM_TAC(GEN_REWRITE_RULE TRY_CONV + [el 1 (CONJUNCTS READ_MEMORY_BYTESIZED_UNSPLIT)]) THEN + RULE_ASSUM_TAC(CONV_RULE + (ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[] THEN POP_ASSUM_LIST(K ALL_TAC) THEN + REWRITE_TAC[CONS_11] THEN REPEAT CONJ_TAC THEN CONV_TAC WORD_BLAST);; + +(* NOTE: This must be kept in sync with the CBMC specification + * in mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h *) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_CORRECT = prove + (`!a rc A1 A2 A3 pc stackpointer returnaddress. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (word_sub stackpointer (word 224),224) /\ + ALLPAIRS nonoverlapping + [(a,600); (word_sub stackpointer (word 224),224)] + [(word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc); (rc,192)] + ==> ensures arm + (\s. aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word pc /\ + read SP s = stackpointer /\ + read X30 s = returnaddress /\ + C_ARGUMENTS [a; rc] s /\ + wordlist_from_memory(a,25) s = A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = A3 /\ + wordlist_from_memory(rc,24) s = round_constants) + (\s. read PC s = returnaddress /\ + wordlist_from_memory(a,25) s = keccak 24 A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = + keccak 24 A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = + keccak 24 A3) + (MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI ,, + MAYCHANGE [memory :> bytes(a,600); + memory :> bytes(word_sub stackpointer (word 224),224)])`, + CONV_TAC LENGTH_SIMPLIFY_CONV THEN + let TWEAK_CONV = + ONCE_DEPTH_CONV + (WORDLIST_FROM_MEMORY_CONV THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) in + CONV_TAC TWEAK_CONV THEN + ARM_ADD_RETURN_STACK_TAC ~pre_post_nsteps:(11,11) KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC + (CONV_RULE TWEAK_CONV (CONV_RULE LENGTH_SIMPLIFY_CONV KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORRECT)) + `[D8; D9; D10; D11; D12; D13; D14; D15; + X19; X20; X21; X22; X23; X24; X25; X26; X27; X28; X29; X30]` 224);; + +(* ------------------------------------------------------------------------- *) +(* Constant-time and memory safety proof. *) +(* ------------------------------------------------------------------------- *) + +needs "s2n_bignum/arm/proofs/consttime.ml";; +needs "mldsa_native/aarch64/proofs/subroutine_signatures.ml";; + +let full_spec,public_vars = mk_safety_spec + ~keep_maychanges:false + (assoc "sha3_keccak3_f1600_alt" subroutine_signatures) + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_CORRECT + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_SAFE = time prove + (`exists f_events. + forall e a rc pc stackpointer returnaddress. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (word_sub stackpointer (word 224),224) /\ + ALLPAIRS nonoverlapping + [a,600; word_sub stackpointer (word 224),224] + [word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc; rc,192] + ==> ensures arm + (\s. + aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word pc /\ + read SP s = stackpointer /\ + read X30 s = returnaddress /\ + C_ARGUMENTS [a; rc] s /\ + read events s = e) + (\s. + read PC s = returnaddress /\ + exists e2. + read events s = APPEND e2 e /\ + e2 = + f_events rc a pc (word_sub stackpointer (word 224)) + returnaddress /\ + memaccess_inbounds e2 + [a,600; rc,192; + word_sub stackpointer (word 224),224] + [a,600; word_sub stackpointer (word 224),224]) + (\s s'. true)`, + ASSERT_CONCL_TAC full_spec THEN + PROVE_SAFETY_SPEC_TAC ~public_vars:public_vars KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC);; diff --git a/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml b/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml index 28c7f0b9d5..fe4d18b7e0 100644 --- a/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml +++ b/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml @@ -373,4 +373,21 @@ let subroutine_signatures = [ [(* temporary buffers *) ]) ); + +("sha3_keccak3_f1600_alt", + ([(*args*) + ("a", "uint64_t[static 75]", (*is const?*)"false"); + ("rc", "uint64_t[static 24]", (*is const?*)"true"); + ], + "void", + [(* input buffers *) + ("a", "75"(* num elems *), 8(* elem bytesize *)); + ("rc", "24"(* num elems *), 8(* elem bytesize *)); + ], + [(* output buffers *) + ("a", "75"(* num elems *), 8(* elem bytesize *)); + ], + [(* temporary buffers *) + ]) +); ];; diff --git a/scripts/autogen b/scripts/autogen index 6f783c32a8..27bbd9f05c 100755 --- a/scripts/autogen +++ b/scripts/autogen @@ -1588,6 +1588,7 @@ def gen_aarch64_keccak_constants_c_file(): yield "#if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \\" + yield " defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \\" yield " !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)" @@ -1602,7 +1603,8 @@ def gen_aarch64_keccak_constants_c_file(): yield "" yield "#else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \\" yield " MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \\" - yield " || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" + yield " || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \\" + yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \\" yield " !MLD_CONFIG_MULTILEVEL_NO_SHARED */" yield "" @@ -1610,7 +1612,8 @@ def gen_aarch64_keccak_constants_c_file(): yield "" yield "#endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \\" yield " MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \\" - yield " || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" + yield " || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \\" + yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \\" yield " !MLD_CONFIG_MULTILEVEL_NO_SHARED) */" yield "" @@ -3061,6 +3064,12 @@ def hol_light_asm_joblist(): f"-Imldsa/src/fips202/native/aarch64/src {aarch64_flags}", "aarch64", ), + ( + "keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S", + "dev/fips202/aarch64/src", + f"-Imldsa/src/fips202/native/aarch64/src {aarch64_flags}", + "aarch64", + ), ( "keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S", "dev/fips202/aarch64/src", @@ -3684,6 +3693,26 @@ def gen_slothy(funcs): print(f"Failed to run SLOTHY on {t}!") exit(1) + # Rename `name` field in YAML block. SLOTHY copies the YAML block of + # its input, so kernels generated from a symbolic/clean source carry + # that source's name (*_clean, *_symbolic). Only those are renamed; + # other kernels keep their name, which need not equal `func` + # (e.g. poly_pointwise_montgomery_aarch64_asm). + out_path = f"{base}/{t}" + out_name = func + with open(out_path, "r") as f: + content = f.read() + new_content = re.sub( + r"^(\s*Name:\s*)\S+_(?:clean|symbolic)\s*$", + lambda m: m.group(1) + out_name, + content, + count=1, + flags=re.MULTILINE, + ) + if new_content != content: + with open(out_path, "w") as f: + f.write(new_content) + class BibliographyEntry: def __init__(self, raw_dict): @@ -5195,6 +5224,10 @@ def _main(): "rej_uniform_aarch64_asm", "rej_uniform_eta2_aarch64_asm", "rej_uniform_eta4_aarch64_asm", + "keccak_f1600_x1_scalar_aarch64_asm", + "keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm", + "keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm", + "keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm", ] parser = argparse.ArgumentParser( diff --git a/scripts/check-contracts b/scripts/check-contracts index e1dcc19b27..f37005658f 100755 --- a/scripts/check-contracts +++ b/scripts/check-contracts @@ -167,7 +167,9 @@ def check_contracts(): DECREASES_EXCEPTIONS = { ("mldsa/src/poly.c", "mld_poly_uniform"): 1, ("mldsa/src/poly.c", "mld_poly_uniform_4x"): 1, + ("mldsa/src/poly.c", "mld_poly_uniform_3x"): 1, ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta_4x"): 1, + ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta_3x"): 1, ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta"): 1, ("mldsa/src/poly_kl.c", "mld_poly_challenge"): 1, } diff --git a/scripts/tests b/scripts/tests index 1f6c5bb06e..88dd4b0674 100755 --- a/scripts/tests +++ b/scripts/tests @@ -1427,6 +1427,7 @@ def cli(): "x2_v84a", "x4_v8a_scalar", "x4_v8a_v84a_scalar", + "x3_v84a_scalar", ], default="auto", type=str, diff --git a/test/abicheck/aarch64/abicheck_aarch64.mk b/test/abicheck/aarch64/abicheck_aarch64.mk index 0e5d2a3028..4403969845 100644 --- a/test/abicheck/aarch64/abicheck_aarch64.mk +++ b/test/abicheck/aarch64/abicheck_aarch64.mk @@ -21,6 +21,7 @@ ABICHECK_REQ_SHA3_FILES := ABICHECK_REQ_NEON_FILES := \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S \ + mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/native/aarch64/src/mldsa_intt_aarch64_asm.S \ @@ -47,6 +48,7 @@ $(ABICHECK_REQ_NEON_OBJS): CFLAGS += -march=armv8-a+simd ABICHECK_REQ_SHA3_FILES := \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S \ + mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S ABICHECK_REQ_SHA3_OBJS := $(call MAKE_OBJS,$(ABICHECK_DIR),$(ABICHECK_REQ_SHA3_FILES)) $(ABICHECK_REQ_SHA3_OBJS): CFLAGS += -march=armv8.4-a+sha3 diff --git a/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c b/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c new file mode 100644 index 0000000000..7bebe6aad9 --- /dev/null +++ b/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_aarch64.h" +#include "../checks_aarch64_all.h" + +#if defined(MLD_SYS_AARCH64) && defined(MLD_SYS_AARCH64_NEON) && \ + defined(__ARM_FEATURE_SHA3) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct aarch64_register_state reg_state; + +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]); + +int check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_x0[600]; /* Three sequential Keccak states (state0[25], + state1[25], state2[25]) */ + MLD_ALIGN uint8_t buf_x1[192]; /* Round constants (24 x uint64_t) */ + + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON)) + { + fprintf(stderr, + "ABI check keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: host " + "lacks AArch64 NEON, skipping\n"); + return MLD_ABICHECK_SKIPPED; + } + + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + fprintf(stderr, + "ABI check keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: host " + "lacks Armv8.4-A SHA3 (eor3, rax1, xar, bcax), skipping\n"); + return MLD_ABICHECK_SKIPPED; + } + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_aarch64_register_state(&input_state); + + randombytes(buf_x0, 600); + randombytes(buf_x1, 192); + + /* Set up register state for function arguments */ + input_state.gpr[0] = (uint64_t)buf_x0; + input_state.gpr[1] = (uint64_t)buf_x1; + + /* Call function through ABI test stub */ + call_stub_aarch64( + &input_state, &output_state, + (void (*)(void))mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm); + + /* Check ABI compliance */ + violations = check_aarch64_aapcs_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf( + stderr, + "ABI test FAILED for keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm " + "(iteration %d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_AARCH64 && MLD_SYS_AARCH64_NEON && __ARM_FEATURE_SHA3 */ diff --git a/test/abicheck/aarch64/checks_aarch64_all.h b/test/abicheck/aarch64/checks_aarch64_all.h index 3411639b80..d0ea90bccf 100644 --- a/test/abicheck/aarch64/checks_aarch64_all.h +++ b/test/abicheck/aarch64/checks_aarch64_all.h @@ -23,6 +23,7 @@ int check_keccak_f1600_x1_scalar_aarch64_asm(void); #if defined(__ARM_FEATURE_SHA3) int check_keccak_f1600_x1_v84a_aarch64_asm(void); int check_keccak_f1600_x2_v84a_aarch64_asm(void); +int check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(void); #endif #endif /* MLD_SYS_AARCH64_NEON */ #if defined(MLD_SYS_AARCH64_NEON) @@ -62,6 +63,8 @@ static const abicheck_entry_t all_checks[] = { check_keccak_f1600_x1_v84a_aarch64_asm}, {"keccak_f1600_x2_v84a_aarch64_asm", check_keccak_f1600_x2_v84a_aarch64_asm}, + {"keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm", + check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm}, #endif #endif /* MLD_SYS_AARCH64_NEON */ #if defined(MLD_SYS_AARCH64_NEON) diff --git a/test/bench/bench_components_mldsa.c b/test/bench/bench_components_mldsa.c index 788e02b7b6..e8e7ee7de5 100644 --- a/test/bench/bench_components_mldsa.c +++ b/test/bench/bench_components_mldsa.c @@ -15,6 +15,10 @@ #include "../../mldsa/src/randombytes.h" #include "hal.h" +/* Batched Keccak permute declarations and lane/way constants for the + * keccakf1600x4_permute and keccakf1600x3_permute benchmarks. */ +#include "../../mldsa/src/fips202/keccakf1600.h" + #ifndef MLD_BENCHMARK_NWARMUP #define MLD_BENCHMARK_NWARMUP 50 #endif @@ -96,10 +100,29 @@ static int bench(void) BENCH("poly_uniform", mld_poly_uniform(&poly_out, seed)) } + /* poly_uniform_4x — the 4-way batched ExpandA (matrix) sampler; x4 + * counterpart to poly_uniform_3x below. Only benchmarked in the x4 build: + * under MLD_CONFIG_USE_X3 its Keccak falls back to a non-native path, so the + * number would not reflect the real x4 kernel. Guard also matches its + * declaration in poly.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_a0, poly_a1, poly_a2, poly_a3; + MLD_ALIGN uint8_t seed_ext4[4][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + CHECK(mld_randombytes((uint8_t *)seed_ext4, sizeof(seed_ext4)) == 0); + BENCH("poly_uniform_4x", mld_poly_uniform_4x(&poly_a0, &poly_a1, &poly_a2, + &poly_a3, seed_ext4)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) */ + BENCH("poly_caddq", mld_poly_caddq((mld_poly *)data0)); - /* poly_uniform_eta_4x — exercises rej_uniform_eta{2,4}_avx2_asm on x86 */ -#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + /* poly_uniform_eta_4x — exercises rej_uniform_eta{2,4}_avx2_asm on x86. + * Not benchmarked with MLD_CONFIG_USE_X3, where the AArch64 backend provides + * the x3 kernel instead and this row would measure a fallback. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ !defined(MLD_CONFIG_NO_KEYPAIR_API) { MLD_ALIGN mld_poly poly_eta0, poly_eta1, poly_eta2, poly_eta3; @@ -108,8 +131,86 @@ static int bench(void) mld_poly_uniform_eta_4x(&poly_eta0, &poly_eta1, &poly_eta2, &poly_eta3, (const uint8_t *)data0, 0, 1, 2, 3)) } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + !MLD_CONFIG_NO_KEYPAIR_API */ + + /* poly_uniform_gamma1_4x — ExpandMask x4 sampler. x4 build only (same reason + * as eta_4x above). Guard matches its declaration in poly_kl.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_g0, poly_g1, poly_g2, poly_g3; + BENCH("poly_uniform_gamma1_4x", + mld_poly_uniform_gamma1_4x(&poly_g0, &poly_g1, &poly_g2, &poly_g3, + (const uint8_t *)data0, 0, 1, 2, 3)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) */ + + /* keccakf1600x4_permute — raw 4-way permutation (four sequential Keccak + * states, one call); x4 counterpart to keccakf1600x3_permute below (per + * state: x4 value / 4 vs x3 value / 3). x4 build only (same reason as + * eta_4x above). Guard matches its declaration in keccakf1600.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ + defined(MLD_UNIT_TEST)) + { + MLD_ALIGN uint64_t x4_state[MLD_KECCAK_LANES * MLD_KECCAK_WAY]; + CHECK(mld_randombytes((uint8_t *)x4_state, sizeof(x4_state)) == 0); + BENCH("keccakf1600x4_permute", mld_keccakf1600x4_permute(x4_state)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST) */ + +#if defined(MLD_CONFIG_USE_X3) + /* + * x3 (3-way) batched Keccak benchmarks, built with MLD_CONFIG_USE_X3: the raw + * permutation and the ExpandA, ExpandS and ExpandMask samplers, to compare + * with the x4 sampler rows above (per poly: x3 value / 3 vs x4 value / 4). + */ + + /* Raw x3 permutation (three sequential Keccak states, one call). */ + { + MLD_ALIGN uint64_t x3_state[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; + CHECK(mld_randombytes((uint8_t *)x3_state, sizeof(x3_state)) == 0); + BENCH("keccakf1600x3_permute", mld_keccakf1600x3_permute(x3_state)) + } + + /* ExpandA x3 sampler (SHAKE128, three matrix entries per call). */ + { + MLD_ALIGN mld_poly poly_a0, poly_a1, poly_a2; + MLD_ALIGN uint8_t seed_ext[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + CHECK(mld_randombytes((uint8_t *)seed_ext, sizeof(seed_ext)) == 0); + BENCH("poly_uniform_3x", + mld_poly_uniform_3x(&poly_a0, &poly_a1, &poly_a2, seed_ext)) + } + + /* ExpandS x3 sampler (SHAKE256, three secret polynomials per call). */ +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) + { + MLD_ALIGN mld_poly poly_s0, poly_s1, poly_s2; + BENCH("poly_uniform_eta_3x", + mld_poly_uniform_eta_3x(&poly_s0, &poly_s1, &poly_s2, + (const uint8_t *)data0, 0, 1, 2)) + } #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_NO_KEYPAIR_API */ + /* ExpandMask x3 sampler (SHAKE256, three gamma1 mask polynomials per call). + */ +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_g0, poly_g1, poly_g2; + BENCH("poly_uniform_gamma1_3x", + mld_poly_uniform_gamma1_3x(&poly_g0, &poly_g1, &poly_g2, + (const uint8_t *)data0, 0, 1, 2)) + } +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST) */ +#endif /* MLD_CONFIG_USE_X3 */ + BENCH("poly_chknorm", chknorm_acc ^= mld_poly_chknorm((const mld_poly *)data0, MLDSA_GAMMA1 - MLDSA_BETA);) diff --git a/test/configs/break_pct_config.h b/test/configs/break_pct_config.h index e435417682..570de15fc8 100644 --- a/test/configs/break_pct_config.h +++ b/test/configs/break_pct_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/configs.yml b/test/configs/configs.yml index 83f2d1985f..c0399d5ea0 100644 --- a/test/configs/configs.yml +++ b/test/configs/configs.yml @@ -28,6 +28,11 @@ configs: defines: MLD_CONFIG_SERIAL_FIPS202_ONLY: true + - path: test/configs/x3_keccak_config.h + description: "Test configuration with the 3-way (x3) batched Keccak path" + defines: + MLD_CONFIG_USE_X3: true + - path: test/configs/custom_randombytes_config.h description: "Test configuration with custom randombytes" defines: diff --git a/test/configs/custom_heap_alloc_config.h b/test/configs/custom_heap_alloc_config.h index b2201f3207..50f63f8f59 100644 --- a/test/configs/custom_heap_alloc_config.h +++ b/test/configs/custom_heap_alloc_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_memcpy_config.h b/test/configs/custom_memcpy_config.h index b2728231ba..6c0d06b106 100644 --- a/test/configs/custom_memcpy_config.h +++ b/test/configs/custom_memcpy_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_memset_config.h b/test/configs/custom_memset_config.h index f0d5b7b214..91cf51bc55 100644 --- a/test/configs/custom_memset_config.h +++ b/test/configs/custom_memset_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_0.h b/test/configs/custom_native_capability_config_0.h index 4cc7e99437..dac6e7863c 100644 --- a/test/configs/custom_native_capability_config_0.h +++ b/test/configs/custom_native_capability_config_0.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_1.h b/test/configs/custom_native_capability_config_1.h index 281f910865..62194dd1bf 100644 --- a/test/configs/custom_native_capability_config_1.h +++ b/test/configs/custom_native_capability_config_1.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_CPUID_AVX2.h b/test/configs/custom_native_capability_config_CPUID_AVX2.h index 6e36b64571..d38a143c19 100644 --- a/test/configs/custom_native_capability_config_CPUID_AVX2.h +++ b/test/configs/custom_native_capability_config_CPUID_AVX2.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h b/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h index 760c7b7492..6a0c084b7f 100644 --- a/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h +++ b/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_randombytes_config.h b/test/configs/custom_randombytes_config.h index 60ee5bc69d..a2424bd9a9 100644 --- a/test/configs/custom_randombytes_config.h +++ b/test/configs/custom_randombytes_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_stdlib_config.h b/test/configs/custom_stdlib_config.h index b4132acc61..b1d76007ed 100644 --- a/test/configs/custom_stdlib_config.h +++ b/test/configs/custom_stdlib_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_zeroize_config.h b/test/configs/custom_zeroize_config.h index 249e22aa86..cbf8ab66f1 100644 --- a/test/configs/custom_zeroize_config.h +++ b/test/configs/custom_zeroize_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/low_signing_bound_config.h b/test/configs/low_signing_bound_config.h index f238b4a25f..9262dc76af 100644 --- a/test/configs/low_signing_bound_config.h +++ b/test/configs/low_signing_bound_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/no_asm_config.h b/test/configs/no_asm_config.h index 03a8dee9e6..ccb07fa695 100644 --- a/test/configs/no_asm_config.h +++ b/test/configs/no_asm_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/serial_fips202_config.h b/test/configs/serial_fips202_config.h index 413495299e..5ab09bcc67 100644 --- a/test/configs/serial_fips202_config.h +++ b/test/configs/serial_fips202_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/test_alloc_config.h b/test/configs/test_alloc_config.h index 441e16682c..7b443121d7 100644 --- a/test/configs/test_alloc_config.h +++ b/test/configs/test_alloc_config.h @@ -417,6 +417,29 @@ struct test_ctx_t; /* Forward declaration */ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/test_sign_hook_config.h b/test/configs/test_sign_hook_config.h index 90f163ca3c..7a8a1b8e6f 100644 --- a/test/configs/test_sign_hook_config.h +++ b/test/configs/test_sign_hook_config.h @@ -421,6 +421,29 @@ struct test_sign_hook_ctx; /* Forward declaration */ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/x3_keccak_config.h b/test/configs/x3_keccak_config.h new file mode 100644 index 0000000000..6c4753702a --- /dev/null +++ b/test/configs/x3_keccak_config.h @@ -0,0 +1,885 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + * + * - [FIPS204_UPDATES] + * FIPS 204 Potential Updates (Errata) + * National Institute of Standards and Technology + * https://csrc.nist.gov/files/pubs/fips/204/final/docs/fips-204-potential-updates.xlsx + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +/* + * Test configuration: Test configuration with the 3-way (x3) batched Keccak + * path + * + * This configuration differs from the default mldsa/mldsa_native_config.h in + * the following places: + * - MLD_CONFIG_USE_X3 + */ + + +#ifndef MLD_CONFIG_H +#define MLD_CONFIG_H + +/** + * MLD_CONFIG_PARAMETER_SET + * + * Specifies the parameter set for ML-DSA + * - MLD_CONFIG_PARAMETER_SET=44 corresponds to ML-DSA-44 + * - MLD_CONFIG_PARAMETER_SET=65 corresponds to ML-DSA-65 + * - MLD_CONFIG_PARAMETER_SET=87 corresponds to ML-DSA-87 + * + * If you want to support multiple parameter sets, build the + * library multiple times and set MLD_CONFIG_MULTILEVEL_BUILD. + * See MLD_CONFIG_MULTILEVEL_BUILD for how to do this while + * minimizing code duplication. + * + * This can also be set using CFLAGS. + */ +#ifndef MLD_CONFIG_PARAMETER_SET +#define MLD_CONFIG_PARAMETER_SET \ + 44 /* Change this for different security strengths */ +#endif + +/** + * MLD_CONFIG_FILE + * + * If defined, this is a header that will be included instead + * of the default configuration file mldsa/mldsa_native_config.h. + * + * When you need to build mldsa-native in multiple configurations, + * using varying MLD_CONFIG_FILE can be more convenient + * than configuring everything through CFLAGS. + * + * To use, MLD_CONFIG_FILE _must_ be defined prior + * to the inclusion of any mldsa-native headers. For example, + * it can be set by passing `-DMLD_CONFIG_FILE="..."` + * on the command line. + */ +/* #define MLD_CONFIG_FILE "mldsa_native_config.h" */ + +/** + * MLD_CONFIG_NAMESPACE_PREFIX + * + * The prefix to use to namespace global symbols from mldsa/. + * + * In a multi-level build, level-dependent symbols will + * additionally be prefixed with the parameter set (44/65/87). + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_NAMESPACE_PREFIX) +#define MLD_CONFIG_NAMESPACE_PREFIX MLD_DEFAULT_NAMESPACE_PREFIX +#endif + +/** + * MLD_CONFIG_MULTILEVEL_BUILD + * + * Set this if the build is part of a multi-level build supporting + * multiple parameter sets. + * + * If you need only a single parameter set, keep this unset. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_BUILD */ + +/** + * MLD_CONFIG_EXTERNAL_API_QUALIFIER + * + * If set, this option provides an additional function + * qualifier to be added to declarations of mldsa-native's + * public API. + * + * The primary use case for this option are single-CU builds + * where the public API exposed by mldsa-native is wrapped by + * another API in the consuming application. In this case, + * even mldsa-native's public API can be marked `static`. + */ +/* #define MLD_CONFIG_EXTERNAL_API_QUALIFIER */ + +/** + * MLD_CONFIG_NO_KEYPAIR_API + * + * By default, mldsa-native includes support for generating key + * pairs. If you don't need this, set MLD_CONFIG_NO_KEYPAIR_API + * to exclude keypair, keypair_internal, + * pk_from_sk, and all internal APIs only needed by + * those functions. + */ +/* #define MLD_CONFIG_NO_KEYPAIR_API */ + +/** + * MLD_CONFIG_NO_SIGN_API + * + * By default, mldsa-native includes support for creating + * signatures. If you don't need this, set MLD_CONFIG_NO_SIGN_API + * to exclude signature, + * signature_extmu, signature_internal, + * signature_pre_hash_internal, + * signature_pre_hash_shake256, and all internal APIs + * only needed by those functions. + */ +/* #define MLD_CONFIG_NO_SIGN_API */ + +/** + * MLD_CONFIG_NO_VERIFY_API + * + * By default, mldsa-native includes support for verifying + * signatures. If you don't need this, set + * MLD_CONFIG_NO_VERIFY_API to exclude verify, + * verify_extmu, verify_internal, + * verify_pre_hash_internal, + * verify_pre_hash_shake256, and all internal APIs + * only needed by those functions. + */ +/* #define MLD_CONFIG_NO_VERIFY_API */ + +/** + * MLD_CONFIG_CORE_API_ONLY + * + * Set this to remove all public APIs except + * keypair_internal, signature_internal, + * and verify_internal. + */ +/* #define MLD_CONFIG_CORE_API_ONLY */ + +/** + * MLD_CONFIG_NO_RANDOMIZED_API + * + * If this option is set, mldsa-native will be built without the + * randomized API functions (keypair, + * signature, and signature_extmu). + * This allows users to build mldsa-native without providing a + * randombytes() implementation if they only need the + * internal deterministic API + * (keypair_internal, signature_internal). + * + * @note This option is incompatible with MLD_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires + * signature(). + */ +/* #define MLD_CONFIG_NO_RANDOMIZED_API */ + +/** + * MLD_CONFIG_CONSTANTS_ONLY + * + * If you only need the size constants (MLDSA_PUBLICKEYBYTES, etc.) + * but no function declarations, set MLD_CONFIG_CONSTANTS_ONLY. + * + * This only affects the public header mldsa_native.h, not + * the implementation. + */ +/* #define MLD_CONFIG_CONSTANTS_ONLY */ + +/** + * MLD_CONFIG_KEYGEN_PCT + * + * Compliance with @[FIPS140_3_IG, p.87] requires a + * Pairwise Consistency Test (PCT) to be carried out on a freshly + * generated keypair before it can be exported. + * + * Set this option if such a check should be implemented. + * In this case, keypair_internal and + * keypair will return MLD_ERR_PCT_FAIL if the + * PCT failed. + * + * @note This feature will drastically lower the performance of + * key generation. + * + * @note This option is incompatible with MLD_CONFIG_NO_SIGN_API + * and MLD_CONFIG_NO_VERIFY_API as the current PCT implementation + * requires signature() and verify(). + */ +/* #define MLD_CONFIG_KEYGEN_PCT */ + +/** + * MLD_CONFIG_CONTEXT_PARAMETER + * + * Set this to add a caller-supplied context parameter to the public API + * functions, which is then forwarded unchanged to the custom callbacks + * (allocation, and signing hooks below). + * + * When this option is set, every public API function gains a trailing + * parameter + * + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE context + * + * as its last argument; its type is configured via + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE (see below). mldsa-native treats this + * value as opaque: it never dereferences it and only passes it on to the + * configurable hook macros. It is meant to carry per-caller state -- e.g. a + * pointer to a memory pool for the allocation hooks, or the resume state for + * the signing hooks -- into those hooks. + * + * When this option is unset (the default), no extra parameter is added and + * the hook macros never receive a context argument. + * + * The hooks that receive the context are the allocation hooks (see + * MLD_CONFIG_CUSTOM_ALLOC_FREE) and the signing hooks (see + * MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH); each is documented with + * its own option below. + */ +/* #define MLD_CONFIG_CONTEXT_PARAMETER */ + +/** + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE + * + * Set this to define the type of the context parameter added by + * MLD_CONFIG_CONTEXT_PARAMETER. It can be any C type usable as a function + * parameter, e.g. `void *` or a pointer to a caller-defined struct such as + * `struct my_ctx *`. + * + * This option must be defined if and only if MLD_CONFIG_CONTEXT_PARAMETER is + * defined; defining one without the other is a compile-time error. + */ +/* #define MLD_CONFIG_CONTEXT_PARAMETER_TYPE void* */ + +/** + * MLD_CONFIG_REDUCE_RAM + * + * Set this to reduce RAM usage. This trades memory for performance. + * + * For expected memory usage, see the MLD_TOTAL_ALLOC_* constants defined in + * mldsa_native.h. + * + * This option is useful for embedded systems with tight RAM constraints but + * relaxed performance requirements. + * + */ +/* #define MLD_CONFIG_REDUCE_RAM */ +/****************************************************************************** + * + * Build-only configuration options + * + * The remaining configurations only affect the build, not the API + * described in mldsa_native.h. + * + *****************************************************************************/ + +/** + * MLD_CONFIG_MULTILEVEL_WITH_SHARED + * + * This is for multi-level builds of mldsa-native only. If you + * need only a single parameter set, keep this unset. + * + * If this is set, all MLD_CONFIG_PARAMETER_SET-independent + * code will be included in the build, including code needed only + * for other parameter sets. + * + * Example: mld_polyw1_pack_88 is only needed for + * MLD_CONFIG_PARAMETER_SET == 44. Yet, if this option is set for a + * build with MLD_CONFIG_PARAMETER_SET == 65/87, it would be included. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_WITH_SHARED */ + +/** + * MLD_CONFIG_MULTILEVEL_NO_SHARED + * + * This is for multi-level builds of mldsa-native only. If you + * need only a single parameter set, keep this unset. + * + * If this is set, no MLD_CONFIG_PARAMETER_SET-independent code + * will be included in the build. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +/** + * MLD_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * + * This is only relevant for single compilation unit (SCU) + * builds of mldsa-native. In this case, it determines whether + * directives defined in parameter-set-independent headers should + * be #undef'ined or not at the end of the SCU file. This is + * needed in multilevel builds. + * + * See examples/multilevel_build_native for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ + +/** + * MLD_CONFIG_USE_NATIVE_BACKEND_ARITH + * + * Determines whether a native arithmetic backend should be used. + * + * The arithmetic backend covers performance-critical functions + * such as the number-theoretic transform (NTT). + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the arithmetic backend to be used is + * determined by MLD_CONFIG_ARITH_BACKEND_FILE: If the latter is + * unset, the default backend for your target architecture + * will be used. If set, it must be the name of a backend metadata + * file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* #define MLD_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif + +/** + * MLD_CONFIG_ARITH_BACKEND_FILE + * + * The arithmetic backend to use. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option + * is ignored. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must + * either be undefined or the filename of an arithmetic backend. + * If unset, the default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLD_CONFIG_ARITH_BACKEND_FILE) +#define MLD_CONFIG_ARITH_BACKEND_FILE "native/meta.h" +#endif + +/** + * MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * + * Determines whether a native FIPS202 backend should be used. + * + * The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is + * the performance bottleneck of SHA3 and SHAKE. + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the FIPS202 backend to be used is + * determined by MLD_CONFIG_FIPS202_BACKEND_FILE: If the latter is + * unset, the default backend for your target architecture + * will be used. If set, it must be the name of a backend metadata + * file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#endif + +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#define MLD_CONFIG_USE_X3 + +/** + * MLD_CONFIG_FIPS202_BACKEND_FILE + * + * The FIPS-202 backend to use. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option + * must either be undefined or the filename of a FIPS202 backend. + * If unset, the default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLD_CONFIG_FIPS202_BACKEND_FILE) +#define MLD_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h" +#endif + +/** + * MLD_CONFIG_FIPS202_CUSTOM_HEADER + * + * Custom header to use for FIPS-202 + * + * This should only be set if you intend to use a custom + * FIPS-202 implementation, different from the one shipped + * with mldsa-native. + * + * If set, it must be the name of a file serving as the + * replacement for mldsa/src/fips202/fips202.h, and exposing + * the same API (see FIPS202.md). + */ +/* #define MLD_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLD_CONFIG_FIPS202X4_CUSTOM_HEADER + * + * Custom header to use for FIPS-202-X4 + * + * This should only be set if you intend to use a custom + * FIPS-202 implementation, different from the one shipped + * with mldsa-native. + * + * If set, it must be the name of a file serving as the + * replacement for mldsa/src/fips202/fips202x4.h, and exposing + * the same API (see FIPS202.md). + */ +/* #define MLD_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLD_CONFIG_CUSTOM_ZEROIZE + * + * In compliance with @[FIPS204, Section 3.6.3], mldsa-native zeroizes + * intermediate buffers before returning from function calls. By default, + * those buffers are allocated from the stack; if MLD_CONFIG_CUSTOM_ALLOC_FREE + * is set, they are (mostly -- few exceptions remain at present) allocated from + * the configured custom allocator. + * + * mldsa-native also zeroizes caller-owned output buffers as needed to uphold + * the API convention that outputs be either unmodified or zeroized upon + * failure. + * + * Set this option and define `mld_zeroize` if you want to use a custom + * method to zeroize intermediate and output buffers. + * + * The default implementation uses SecureZeroMemory on Windows and a + * memset + compiler barrier otherwise. If neither of those is available on + * the target platform, compilation will fail, and you will need to use + * MLD_CONFIG_CUSTOM_ZEROIZE to provide a custom implementation of + * `mld_zeroize()`. + * + * @warning + * The zeroization conducted by mldsa-native reduces the likelihood of data + * leaking on the stack or custom allocators, but it does not eliminate it. + * For example, the C standard makes no guarantee about where a compiler + * allocates local structures and whether/where it makes copies of them. + * Also, in addition to entire structures, there may also be potentially + * exploitable leakage of individual values on the stack. If you need + * bullet-proof zeroization of the stack, you need to consider additional + * measures instead of what this feature provides. In this case, you can + * set mld_zeroize to a no-op. Note that in this case you are also responsible + * for zeroizing output buffers upon failure. + */ +/* #define MLD_CONFIG_CUSTOM_ZEROIZE + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void mld_zeroize(void *ptr, size_t len) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_RANDOMBYTES + * + * mldsa-native does not provide a secure randombytes + * implementation. Such an implementation has to be provided by + * the consumer. + * + * If this option is not set, mldsa-native expects a function + * int randombytes(uint8_t *out, size_t outlen). + * + * Set this option and define `mld_randombytes` if you want to + * use a custom method to sample randombytes with a different name + * or signature. + */ +/* #define MLD_CONFIG_CUSTOM_RANDOMBYTES + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE int mld_randombytes(uint8_t *ptr, size_t len) + { + ... your implementation ... + return 0; + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_CAPABILITY_FUNC + * + * mldsa-native backends may rely on specific hardware features. + * Those backends will only be included in an mldsa-native build + * if support for the respective features is enabled at + * compile-time. However, when building for a heterogeneous set + * of CPUs to run the resulting binary/library on, feature + * detection at _runtime_ is needed to decide whether a backend + * can be used or not. + * + * Set this option and define `mld_sys_check_capability` if you + * want to use a custom method to dispatch between implementations. + * + * Return value 1 indicates that a capability is supported. + * Return value 0 indicates that a capability is not supported. + * + * If this option is not set, mldsa-native uses compile-time + * feature detection only to decide which backend to use. + * + * If you compile mldsa-native on a system with different + * capabilities than the system that the resulting binary/library + * will be run on, you must use this option. + */ +/* #define MLD_CONFIG_CUSTOM_CAPABILITY_FUNC + static MLD_INLINE int mld_sys_check_capability(mld_sys_cap cap) + { + ... your implementation ... + } +*/ + +/** + * MLD_CONFIG_CUSTOM_ALLOC_FREE + * + * Set this option and define `MLD_CUSTOM_ALLOC` and + * `MLD_CUSTOM_FREE` if you want to use custom allocation for + * large local structures or buffers. + * + * By default, all buffers/structures are allocated on the stack. + * If this option is set, most of them will be allocated via + * MLD_CUSTOM_ALLOC. + * + * Parameters to MLD_CUSTOM_ALLOC: + * - T* v: Target pointer to declare. + * - T: Type of structure to be allocated + * - N: Number of elements to be allocated. + * + * Parameters to MLD_CUSTOM_FREE: + * - T* v: Target pointer to free. May be NULL. + * - T: Type of structure to be freed. + * - N: Number of elements to be freed. + * + * @note Even if this option is set, some allocations further down + * the call stack will still be made from the stack. Those will + * likely be added to the scope of this option in the future. + * + * @note MLD_CUSTOM_ALLOC need not guarantee a successful + * allocation nor include error handling. Upon failure, the + * target pointer should simply be set to NULL. The calling + * code will handle this case and invoke MLD_CUSTOM_FREE. + */ +/* #define MLD_CONFIG_CUSTOM_ALLOC_FREE + #if !defined(__ASSEMBLER__) + #include + #define MLD_CUSTOM_ALLOC(v, T, N) \ + T* (v) = (T *)aligned_alloc(MLD_DEFAULT_ALIGN, \ + MLD_ALIGN_UP(sizeof(T) * (N))) + #define MLD_CUSTOM_FREE(v, T, N) free(v) + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_MEMCPY + * + * Set this option and define `mld_memcpy` if you want to + * use a custom method to copy memory instead of the standard + * library memcpy function. + * + * The custom implementation must have the same signature and + * behavior as the standard memcpy function: + * void *mld_memcpy(void *dest, const void *src, size_t n) + */ +/* #define MLD_CONFIG_CUSTOM_MEMCPY + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void *mld_memcpy(void *dest, const void *src, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_MEMSET + * + * Set this option and define `mld_memset` if you want to + * use a custom method to set memory instead of the standard + * library memset function. + * + * The custom implementation must have the same signature and + * behavior as the standard memset function: + * void *mld_memset(void *s, int c, size_t n) + */ +/* #define MLD_CONFIG_CUSTOM_MEMSET + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void *mld_memset(void *s, int c, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_INTERNAL_API_QUALIFIER + * + * If set, this option provides an additional qualifier + * to be added to declarations of internal API functions and data. + * + * The primary use case for this option are single-CU builds, + * in which case this option can be set to `static`. + */ +/* #define MLD_CONFIG_INTERNAL_API_QUALIFIER */ + +/** + * MLD_CONFIG_CT_TESTING_ENABLED + * + * If set, mldsa-native annotates data as secret / public using + * valgrind's annotations VALGRIND_MAKE_MEM_UNDEFINED and + * VALGRIND_MAKE_MEM_DEFINED, enabling various checks for secret- + * dependent control flow of variable time execution (depending + * on the exact version of valgrind installed). + */ +/* #define MLD_CONFIG_CT_TESTING_ENABLED */ + +/** + * MLD_CONFIG_NO_ASM + * + * If this option is set, mldsa-native will be built without + * use of native code or inline assembly. + * + * By default, inline assembly is used to implement value barriers. + * Without inline assembly, mldsa-native will use a global volatile + * 'opt blocker' instead; see ct.h. + * + * Inline assembly is also used to implement a secure zeroization + * function on non-Windows platforms. If this option is set and + * the target platform is not Windows, you MUST set + * MLD_CONFIG_CUSTOM_ZEROIZE and provide a custom zeroization + * function. + * + * If this option is set, MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 and + * MLD_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no + * native backends will be used. + */ +/* #define MLD_CONFIG_NO_ASM */ + +/** + * MLD_CONFIG_NO_ASM_VALUE_BARRIER + * + * If this option is set, mldsa-native will be built without + * use of native code or inline assembly for value barriers. + * + * By default, inline assembly (if available) is used to implement + * value barriers. + * Without inline assembly, mldsa-native will use a global volatile + * 'opt blocker' instead; see ct.h. + */ +/* #define MLD_CONFIG_NO_ASM_VALUE_BARRIER */ + +/** + * MLD_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + * + * If this option is set, the user must provide a runtime + * function `static inline int mld_break_pct() { ... }` to + * indicate whether the PCT should be made fail. + * + * This option only has an effect if MLD_CONFIG_KEYGEN_PCT is set. + */ +/* #define MLD_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + #if !defined(__ASSEMBLER__) + #include "src/src.h" + static MLD_INLINE int mld_break_pct(void) + { + ... return 0/1 depending on whether PCT should be broken ... + } + #endif +*/ + +/** + * MLD_CONFIG_MAX_SIGNING_ATTEMPTS + * + * Upper bound on the number of rejection-sampling iterations + * performed by ML-DSA signing (@[FIPS204, Algorithm 7]). + * + * If a valid signature is not produced within this many + * attempts, signing returns MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED. + * This is useful in timing-sensitive environments that + * require a deterministic worst-case bound on signing time. + * + * For FIPS 204 compliance, this value MUST be at least 821, + * cf. @[FIPS204, Appendix C] and @[FIPS204_UPDATES], which is + * chosen so that the signing failure rate is < 2^{-256}. + * + * Default: Largest possible value before internal counters + * would overflow. This is larger than the FIPS204 bound. + * + * In particular, in the default configuration, the signing + * failure rate is < 2^{-256}. + */ +/* #define MLD_CONFIG_MAX_SIGNING_ATTEMPTS 821 */ + +/** + * MLD_CONFIG_SERIAL_FIPS202_ONLY + * + * Set this to use a FIPS202 implementation with global state + * that supports only one active Keccak computation at a time + * (e.g. some hardware accelerators). + * + * If this option is set, ML-DSA will use FIPS202 operations + * serially, ensuring that only one SHAKE context is active + * at any given time. + * + * This allows offloading Keccak computations to a hardware + * accelerator that holds only a single Keccak state locally, + * rather than requiring support for multiple concurrent + * Keccak states. + * + * @note Depending on the target CPU, this may reduce + * performance when using software FIPS202 implementations. + * Only enable this when you have to. + */ +/* #define MLD_CONFIG_SERIAL_FIPS202_ONLY */ + +/** + * Signing hooks: MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH + * + * Three optional, independent hooks into the ML-DSA signing rejection-sampling + * loop. Each is enabled by defining the matching option, in which case the + * integration must provide the corresponding function. If a hook needs + * per-operation state, enable MLD_CONFIG_CONTEXT_PARAMETER; the context is then + * appended as the last argument. + * + * @warning This feature is experimental. Its scope, configuration and + * function signatures may change at any time, including after v2. + * + * Enabling any of the hooks requires MLD_CONFIG_NO_RANDOMIZED_API (restricting + * the public API to deterministic operations). This is because the restartable + * signing as enabled by the signing hooks only produces the uninterrupted + * signature when the randomness is fixed across calls. A logging-only use + * (attempt always returns 0; resume/finish merely observe) would be safe with + * the randomized API too, but for now the requirement is imposed uniformly on + * all three hooks. + * + * Note: Randomized signing is a shim wrapper around deterministic signing, and + * all helper functions you need to build it are exposed publicly. Thus, if you + * need a restartable, randomized signing operation, you can build your own by + * replicating the logic and adding the RNG seed to the restart context. In this + * case, please also consider letting the mldsa-native maintainers know of your + * need for randomized, restartable signing, so the feature can be appropriately + * prioritized. + * + * - MLD_CONFIG_SIGN_HOOK_ATTEMPT: int mld_sign_hook_attempt(attempt[, ctxt]) + * Called before each attempt. Returns 0 to proceed, or non-zero to pause: + * signing then returns MLD_ERR_SIGNING_PAUSED with `attempt` as the resume + * point (needs MLD_CONFIG_SIGN_HOOK_RESUME to resume; otherwise just aborts). + * Always returning 0 makes it a logging/benchmarking hook. + * + * - MLD_CONFIG_SIGN_HOOK_RESUME: uint16_t mld_sign_hook_resume([ctxt]) + * Returns the attempt to resume from (0 for a fresh operation), i.e. the one + * recorded when a previous call paused. + * + * - MLD_CONFIG_SIGN_HOOK_FINISH: void mld_sign_hook_finish(attempt[, ctxt]) + * Called on success with the succeeding attempt. Observe-only. + * + * When an option is unset, the hook is a no-op (resume to 0, attempt proceeds), + * i.e. ordinary one-shot signing. + * + * Independent of MLD_CONFIG_MAX_SIGNING_ATTEMPTS, which is a static upper bound + * on the number of signing attempts. + * + * See test/src/test_sign_hook.c for a worked example using all three. + */ +/* #define MLD_CONFIG_SIGN_HOOK_RESUME + #define MLD_CONFIG_SIGN_HOOK_ATTEMPT + #define MLD_CONFIG_SIGN_HOOK_FINISH + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLD_INLINE uint16_t mld_sign_hook_resume(void) + { + ... return the attempt to resume from ... + } + static MLD_INLINE int mld_sign_hook_attempt(uint16_t attempt) + { + ... return non-zero to pause here; for resume, store attempt ... + return 0; + } + static MLD_INLINE void mld_sign_hook_finish(uint16_t attempt) + { + ... mark the operation complete (attempt = successful attempt) ... + } + #endif +*/ + +/************************* Config internals ********************************/ + +/* Default namespace + * + * Don't change this. If you need a different namespace, re-define + * MLD_CONFIG_NAMESPACE_PREFIX above instead, and remove the following. + * + * The default MLDSA namespace is + * + * PQCP_MLDSA_NATIVE_MLDSA_ + * + * e.g., PQCP_MLDSA_NATIVE_MLDSA44_ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_BUILD) +/* In a multi-level build the parameter set is appended by the namespacing + * machinery, so the default prefix must not embed it. */ +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA +#elif MLD_CONFIG_PARAMETER_SET == 44 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA44 +#elif MLD_CONFIG_PARAMETER_SET == 65 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA65 +#elif MLD_CONFIG_PARAMETER_SET == 87 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA87 +#endif + +#endif /* !MLD_CONFIG_H */ diff --git a/test/mk/components.mk b/test/mk/components.mk index c3a51aadb9..93f85f5fad 100644 --- a/test/mk/components.mk +++ b/test/mk/components.mk @@ -260,6 +260,7 @@ ABICHECK_ASM_CFLAGS := \ -DMLD_FIPS202_AARCH64_NEED_X1_SCALAR \ -DMLD_FIPS202_AARCH64_NEED_X1_V84A \ -DMLD_FIPS202_AARCH64_NEED_X2_V84A \ + -DMLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID \ -DMLD_FIPS202_ARMV81M_NEED_X4 \