From 772e4369388ca742aff6462d980d5a78ec71c7cc Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Tue, 20 Oct 2026 08:59:28 -0700 Subject: [PATCH 1/8] FIPS202/AArch64: Add symbolic Keccak sources and SLOTHY Makefile Port the symbolic Keccak-f1600 sources and SLOTHY recipes from mlkem-native, so that `autogen --slothy` regenerates the existing x1 and x4 kernels and the SLOTHY CI job tests them. Signed-off-by: Soumya Bhattacharya --- .github/workflows/slothy.yml | 1 + dev/fips202/aarch64/src/Makefile | 112 ++ .../keccak_f1600_x1_scalar_symbolic.S | 527 +++++++++ .../keccak_f1600_x4_v8a_scalar_hybrid_clean.S | 1031 ++++++++++++++++ ...ak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S | 1052 +++++++++++++++++ scripts/autogen | 23 + 6 files changed, 2746 insertions(+) create mode 100644 dev/fips202/aarch64/src/Makefile create mode 100644 dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S create mode 100644 dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S create mode 100644 dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S diff --git a/.github/workflows/slothy.yml b/.github/workflows/slothy.yml index 85adf720b7..ba29d88f58 100644 --- a/.github/workflows/slothy.yml +++ b/.github/workflows/slothy.yml @@ -61,6 +61,7 @@ jobs: dev/aarch64_clean/** dev/aarch64_opt/** dev/fips202/aarch64/** + dev/fips202/aarch64_symbolic/** mldsa/src/native/aarch64/** mldsa/src/fips202/native/aarch64/** flake.nix diff --git a/dev/fips202/aarch64/src/Makefile b/dev/fips202/aarch64/src/Makefile new file mode 100644 index 0000000000..697ec4ca3f --- /dev/null +++ b/dev/fips202/aarch64/src/Makefile @@ -0,0 +1,112 @@ +# Copyright (c) The mlkem-native project authors +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +.PHONY: all purge +.DEFAULT_GOAL := all + +keccak_f1600_x1_scalar_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S + # The last two configuration options are for fast testing (incl. CI). + # Remove when building a production version. + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c constraints.allow_spills \ + -c constraints.minimize_spills \ + -s mld_keccak_f1600_x1_scalar_loop \ + -e mld_keccak_f1600_x1_scalar_end_loop \ + -c constraints.functional_only \ + -c objective_lower_bound=24 + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c variable_size \ + -c inputs_are_outputs \ + -c outputs="[hint_STACK_LOC_COUNT]" \ + -c constraints.stalls_first_attempt=64 \ + -c constraints.allow_spills \ + -c constraints.minimize_spills \ + -s mld_keccak_f1600_x1_scalar_initial_start \ + -e mld_keccak_f1600_x1_scalar_loop \ + -c constraints.functional_only \ + -c objective_lower_bound=24 + +keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x4_v8a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -c absorb_spills=False \ + -s mld_keccak_f1600_x4_v8a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x4_v8a_scalar_hybrid_loop_end + +keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -c absorb_spills=False \ + -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end + +ALL=keccak_f1600_x1_scalar_aarch64_asm.S \ + keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S \ + keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S + +all: $(ALL) + +purge: + rm -rf $(ALL) diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S new file mode 100644 index 0000000000..94c1d3dc47 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x1_scalar_symbolic.S @@ -0,0 +1,527 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// + +/*yaml + Name: keccak_f1600_x1_scalar_symbolic + Description: AArch64 scalar symbolic implementation of Keccak-f[1600] permutation for single state (clean version for optimization) + Signature: void mld_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t *state + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 128 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_rc .req x1 + const_addr .req x26 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + Aba .req x1 + Abe .req x6 + Abi .req x11 + Abo .req x16 + Abu .req x21 + Aga .req x2 + Age .req x7 + Agi .req x12 + Ago .req x17 + Agu .req x22 + Aka .req x3 + Ake .req x8 + Aki .req x13 + Ako .req x28 + Aku .req x23 + Ama .req x4 + Ame .req x9 + Ami .req x14 + Amo .req x19 + Amu .req x24 + Asa .req x5 + Ase .req x10 + Asi .req x15 + Aso .req x20 + Asu .req x25 + +/************************ MACROS ****************************/ + +/* Allocate a large number of stack slots by default so one can quickly + * check functional correctness by stopping SLOTHY's register allocation + * and stack spilling process while still using a large number of spills. + * + * NOTE: + * This should to be manually trimmed to the number of stack locations + * actually used after running SLOTHY. + */ +#define STACK_LOCS 40 + +// GPRs (16*6), count (8), const (8), input (8), padding (8) +#define STACK_SIZE (16*6 + 3*8 + 8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (3*8+8) +#define STACK_LOC_INPUT (0*8) +#define STACK_LOC_CONST (1*8) +#define STACK_LOC_COUNT (2*8) +#define STACK_LOC_MISC (16*6 + 4*8) + +#define STACK_LOC_0 ((STACK_LOC_MISC) + 0*8) +#define STACK_LOC_1 ((STACK_LOC_MISC) + 1*8) +#define STACK_LOC_2 ((STACK_LOC_MISC) + 2*8) +#define STACK_LOC_3 ((STACK_LOC_MISC) + 3*8) +#define STACK_LOC_4 ((STACK_LOC_MISC) + 4*8) +#define STACK_LOC_5 ((STACK_LOC_MISC) + 5*8) +#define STACK_LOC_6 ((STACK_LOC_MISC) + 6*8) +#define STACK_LOC_7 ((STACK_LOC_MISC) + 7*8) +#define STACK_LOC_8 ((STACK_LOC_MISC) + 8*8) +#define STACK_LOC_9 ((STACK_LOC_MISC) + 9*8) +#define STACK_LOC_10 ((STACK_LOC_MISC) + 10*8) +#define STACK_LOC_11 ((STACK_LOC_MISC) + 11*8) +#define STACK_LOC_12 ((STACK_LOC_MISC) + 12*8) +#define STACK_LOC_13 ((STACK_LOC_MISC) + 13*8) +#define STACK_LOC_14 ((STACK_LOC_MISC) + 14*8) +#define STACK_LOC_15 ((STACK_LOC_MISC) + 15*8) +#define STACK_LOC_16 ((STACK_LOC_MISC) + 16*8) +#define STACK_LOC_17 ((STACK_LOC_MISC) + 17*8) +#define STACK_LOC_18 ((STACK_LOC_MISC) + 18*8) +#define STACK_LOC_19 ((STACK_LOC_MISC) + 19*8) +#define STACK_LOC_20 ((STACK_LOC_MISC) + 20*8) +#define STACK_LOC_21 ((STACK_LOC_MISC) + 21*8) +#define STACK_LOC_22 ((STACK_LOC_MISC) + 22*8) +#define STACK_LOC_23 ((STACK_LOC_MISC) + 23*8) +#define STACK_LOC_24 ((STACK_LOC_MISC) + 24*8) +#define STACK_LOC_25 ((STACK_LOC_MISC) + 25*8) +#define STACK_LOC_26 ((STACK_LOC_MISC) + 26*8) +#define STACK_LOC_27 ((STACK_LOC_MISC) + 27*8) +#define STACK_LOC_28 ((STACK_LOC_MISC) + 28*8) +#define STACK_LOC_29 ((STACK_LOC_MISC) + 29*8) +#define STACK_LOC_30 ((STACK_LOC_MISC) + 30*8) +#define STACK_LOC_31 ((STACK_LOC_MISC) + 31*8) +#define STACK_LOC_32 ((STACK_LOC_MISC) + 32*8) +#define STACK_LOC_33 ((STACK_LOC_MISC) + 33*8) +#define STACK_LOC_34 ((STACK_LOC_MISC) + 34*8) +#define STACK_LOC_35 ((STACK_LOC_MISC) + 35*8) +#define STACK_LOC_36 ((STACK_LOC_MISC) + 36*8) +#define STACK_LOC_37 ((STACK_LOC_MISC) + 37*8) +#define STACK_LOC_38 ((STACK_LOC_MISC) + 38*8) +#define STACK_LOC_39 ((STACK_LOC_MISC) + 39*8) + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +.macro keccak_f1600_round_initial + eor5 X, Ama, Asa, Aba, Aga, Aka + eor5 X, Ame, Ase, Abe, Age, Ake + eor5 X, Ami, Asi, Abi, Agi, Aki + eor5 X, Amo, Aso, Abo, Ago, Ako + eor5 X, Amu, Asu, Abu, Agu, Aku + + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + + eor X, Aba, X + eor X, Abi, X + eor X, Aki, X + eor X, Ako, X + eor X, Amu, X + eor X, Aso, X + eor X, Ama, X + eor X, Abe, X + eor X, Ago, X + eor X, Ame, X + eor X, Agi, X + eor X, Aka, X + eor X, Abo, X + eor X, Amo, X + eor X, Ami, X + eor X, Ake, X + eor X, Agu, X + eor X, Asi, X + eor X, Aku, X + eor X, Asa, X + eor X, Abu, X + eor X, Asu, X + eor X, Ase, X + eor X, Aga, X + eor X, Age, X + + ldr X, [sp, #STACK_LOC_CONST] + ldr X, [X] + mov X, #1 + str X, [sp, #STACK_LOC_COUNT] // @slothy:writes=STACK_LOC_COUNT + + chi_step_ror Aga, X, X, X, 47, 39 + chi_step_ror Age, X, X, X, 42, 25 + chi_step_ror Agi, X, X, X, 16, 58 + chi_step_ror Ago, X, X, X, 31, 47 + chi_step_ror Agu, X, X, X, 56, 23 + chi_step_ror Aka, X, X, X, 19, 24 + chi_step_ror Ake, X, X, X, 47, 2 + chi_step_ror Aki, X, X, X, 10, 57 + chi_step_ror Ako, X, X, X, 47, 57 + chi_step_ror Aku, X, X, X, 5, 52 + chi_step_ror Ama, X, X, X, 38, 47 + chi_step_ror Ame, X, X, X, 5, 43 + chi_step_ror Ami, X, X, X, 41, 46 + chi_step_ror Amo, X, X, X, 35, 12 + chi_step_ror Amu, X, X, X, 9, 44 + chi_step_ror Asa, X, X, X, 48, 41 + chi_step_ror Ase, X, X, X, 2, 50 + chi_step_ror Asi, X, X, X, 25, 27 + chi_step_ror Aso, X, X, X, 60, 21 + chi_step_ror Asu, X, X, X, 57, 53 + chi_step_ror2 Aba, X, X, X, 63, 21 + chi_step_ror Abe, X, X, X, 42, 41 + chi_step_ror Abi, X, X, X, 57, 35 + chi_step_ror Abo, X, X, X, 50, 43 + chi_step_ror Abu, X, X, X, 44, 30 + + eor Aba, Aba, X + +.endm + +.macro keccak_f1600_round_noninitial + + eor X, Aba, Aga, ror #61 + eor X, X, Ama, ror #54 + eor X, X, Aka, ror #39 + eor X, X, Asa, ror #25 + + eor X, Ake, Ame, ror #57 + eor X, X, Abe, ror #51 + eor X, X, Ase, ror #31 + eor X, X, Age, ror #27 + + eor X, Asi, Abi, ror #52 + eor X, X, Aki, ror #48 + eor X, X, Ami, ror #10 + eor X, X, Agi, ror #5 + + eor X, Abo, Ako, ror #63 + eor X, X, Amo, ror #37 + eor X, X, Ago, ror #36 + eor X, X, Aso, ror #2 + + eor X, Aku, Agu, ror #50 + eor X, X, Amu, ror #34 + eor X, X, Abu, ror #26 + eor X, X, Asu, ror #15 + + eor X, X, X, ror #61 + ror X, X, #62 + eor X, X, X, ror #57 + ror X, X, #58 + eor X, X, X, ror #55 + ror X, X, #56 + eor X, X, X, ror #63 + eor X, X, X, ror #63 + + eor X, X, Aba + eor X, X, Abi, ror #50 + eor X, X, Aki, ror #46 + eor X, X, Ako, ror #63 + eor X, X, Amu, ror #28 + eor X, X, Aso, ror #2 + eor X, X, Ama, ror #54 + eor X, X, Abe, ror #43 + eor X, X, Ago, ror #36 + eor X, X, Ame, ror #49 + eor X, X, Agi, ror #3 + eor X, X, Aka, ror #39 + eor X, X, Abo + eor X, X, Amo, ror #37 + eor X, X, Ami, ror #8 + eor X, X, Ake, ror #56 + eor X, X, Agu, ror #44 + eor X, X, Asi, ror #62 + eor X, X, Aku, ror #58 + eor X, X, Asa, ror #25 + eor X, X, Abu, ror #20 + eor X, X, Asu, ror #9 + eor X, X, Ase, ror #23 + eor X, X, Aga, ror #61 + eor X, X, Age, ror #19 + + ldr X, [sp, #STACK_LOC_CONST] + ldr X, [sp, #STACK_LOC_COUNT] // @slothy:reads=STACK_LOC_COUNT + ldr X, [X, X, LSL #3] + add X, X, #1 + cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + str X, [sp, #STACK_LOC_COUNT] // @slothy:writes=STACK_LOC_COUNT + + chi_step_ror Aga, X, X, X, 47, 39 + chi_step_ror Age, X, X, X, 42, 25 + chi_step_ror Agi, X, X, X, 16, 58 + chi_step_ror Ago, X, X, X, 31, 47 + chi_step_ror Agu, X, X, X, 56, 23 + chi_step_ror Aka, X, X, X, 19, 24 + chi_step_ror Ake, X, X, X, 47, 2 + chi_step_ror Aki, X, X, X, 10, 57 + chi_step_ror Ako, X, X, X, 47, 57 + chi_step_ror Aku, X, X, X, 5, 52 + chi_step_ror Ama, X, X, X, 38, 47 + chi_step_ror Ame, X, X, X, 5, 43 + chi_step_ror Ami, X, X, X, 41, 46 + chi_step_ror Amo, X, X, X, 35, 12 + chi_step_ror Amu, X, X, X, 9, 44 + chi_step_ror Asa, X, X, X, 48, 41 + chi_step_ror Ase, X, X, X, 2, 50 + chi_step_ror Asi, X, X, X, 25, 27 + chi_step_ror Aso, X, X, X, 60, 21 + chi_step_ror Asu, X, X, X, 57, 53 + chi_step_ror2 Aba, X, X, X, 63, 21 + chi_step_ror Abe, X, X, X, 42, 41 + chi_step_ror Abi, X, X, X, 57, 35 + chi_step_ror Abo, X, X, X, 50, 43 + chi_step_ror Abu, X, X, X, 44, 30 + + eor Aba, Aba, X +.endm + +.macro load_state + ldp Aba, Abe, [input_addr, #(1*8*0)] + ldp Abi, Abo, [input_addr, #(1*8*2)] + ldp Abu, Aga, [input_addr, #(1*8*4)] + ldp Age, Agi, [input_addr, #(1*8*6)] + ldp Ago, Agu, [input_addr, #(1*8*8)] + ldp Aka, Ake, [input_addr, #(1*8*10)] + ldp Aki, Ako, [input_addr, #(1*8*12)] + ldp Aku, Ama, [input_addr, #(1*8*14)] + ldp Ame, Ami, [input_addr, #(1*8*16)] + ldp Amo, Amu, [input_addr, #(1*8*18)] + ldp Asa, Ase, [input_addr, #(1*8*20)] + ldp Asi, Aso, [input_addr, #(1*8*22)] + ldr Asu, [input_addr, #(1*8*24)] +.endm + +.macro store_state + stp Aba, Abe, [input_addr, #(1*8*0)] + stp Abi, Abo, [input_addr, #(1*8*2)] + stp Abu, Aga, [input_addr, #(1*8*4)] + stp Age, Agi, [input_addr, #(1*8*6)] + stp Ago, Agu, [input_addr, #(1*8*8)] + stp Aka, Ake, [input_addr, #(1*8*10)] + stp Aki, Ako, [input_addr, #(1*8*12)] + stp Aku, Ama, [input_addr, #(1*8*14)] + stp Ame, Ami, [input_addr, #(1*8*16)] + stp Amo, Amu, [input_addr, #(1*8*18)] + stp Asa, Ase, [input_addr, #(1*8*20)] + stp Asi, Aso, [input_addr, #(1*8*22)] + str Asu, [input_addr, #(1*8*24)] +.endm + +.macro final_rotate + ror Abe, Abe,#(64-21) + ror Abi, Abi,#(64-14) + ror Abu, Abu,#(64-44) + ror Aga, Aga,#(64-3) + ror Age, Age,#(64-45) + ror Agi, Agi,#(64-61) + ror Ago, Ago,#(64-28) + ror Agu, Agu,#(64-20) + ror Aka, Aka,#(64-25) + ror Ake, Ake,#(64-8) + ror Aki, Aki,#(64-18) + ror Ako, Ako,#(64-1) + ror Aku, Aku,#(64-6) + ror Ama, Ama,#(64-10) + ror Ame, Ame,#(64-15) + ror Ami, Ami,#(64-56) + ror Amo, Amo,#(64-27) + ror Amu, Amu,#(64-36) + ror Asa, Asa,#(64-39) + ror Ase, Ase,#(64-41) + ror Asi, Asi,#(64-2) + ror Aso, Aso,#(64-62) + ror Asu, Asu,#(64-55) +.endm + +#define KECCAK_F1600_ROUNDS 24 + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm) + alloc_stack + save_gprs + +mld_keccak_f1600_x1_scalar_initial: + mov const_addr, input_rc + str input_rc, [sp, #STACK_LOC_CONST] + load_state + str input_addr, [sp, #STACK_LOC_INPUT] // @slothy:writes=STACK_LOC_INPUT + +mld_keccak_f1600_x1_scalar_initial_start: + keccak_f1600_round_initial + +mld_keccak_f1600_x1_scalar_loop: + keccak_f1600_round_noninitial +mld_keccak_f1600_x1_scalar_end_loop: + ble mld_keccak_f1600_x1_scalar_loop + +mld_keccak_f1600_x1_scalar_final: + final_rotate + ldr input_addr, [sp, #STACK_LOC_INPUT] // @slothy:reads=STACK_LOC_INPUT + store_state +mld_keccak_f1600_x1_scalar_end_final: + + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_rc + .unreq const_addr + .unreq Aba + .unreq Abe + .unreq Abi + .unreq Abo + .unreq Abu + .unreq Aga + .unreq Age + .unreq Agi + .unreq Ago + .unreq Agu + .unreq Aka + .unreq Ake + .unreq Aki + .unreq Ako + .unreq Aku + .unreq Ama + .unreq Ame + .unreq Ami + .unreq Amo + .unreq Amu + .unreq Asa + .unreq Ase + .unreq Asi + .unreq Aso + .unreq Asu + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_LOC_INPUT +#undef STACK_LOC_CONST +#undef STACK_LOC_COUNT +#undef STACK_LOC_MISC +#undef STACK_LOC_0 +#undef STACK_LOC_1 +#undef STACK_LOC_2 +#undef STACK_LOC_3 +#undef STACK_LOC_4 +#undef STACK_LOC_5 +#undef STACK_LOC_6 +#undef STACK_LOC_7 +#undef STACK_LOC_8 +#undef STACK_LOC_9 +#undef STACK_LOC_10 +#undef STACK_LOC_11 +#undef STACK_LOC_12 +#undef STACK_LOC_13 +#undef STACK_LOC_14 +#undef STACK_LOC_15 +#undef STACK_LOC_16 +#undef STACK_LOC_17 +#undef STACK_LOC_18 +#undef STACK_LOC_19 +#undef STACK_LOC_20 +#undef STACK_LOC_21 +#undef STACK_LOC_22 +#undef STACK_LOC_23 +#undef STACK_LOC_24 +#undef STACK_LOC_25 +#undef STACK_LOC_26 +#undef STACK_LOC_27 +#undef STACK_LOC_28 +#undef STACK_LOC_29 +#undef STACK_LOC_30 +#undef STACK_LOC_31 +#undef STACK_LOC_32 +#undef STACK_LOC_33 +#undef STACK_LOC_34 +#undef STACK_LOC_35 +#undef STACK_LOC_36 +#undef STACK_LOC_37 +#undef STACK_LOC_38 +#undef STACK_LOC_39 +#undef KECCAK_F1600_ROUNDS + +/* simpasm: footer-start */ +#endif /* MLD_FIPS202_AARCH64_NEED_X1_SCALAR && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..fe6e6a0b04 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_scalar_hybrid_clean.S @@ -0,0 +1,1031 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for four sequential states (clean version for optimization) + Signature: void mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t *state + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + outer .req x30 + cur_const .req x26 + + /* Mapping of Kecck-f1600 SIMD state to vector registers + * at the beginning and end of each round. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Unused temporary */ + vtmp .req v31 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m1 d, s0, s1, s2 + eor \d\().16b, \s0\().16b, \s1\().16b + eor \d\().16b, \d\().16b, \s2\().16b +.endm + +.macro rax1_m1 d, s0, s1 + add vtmp.2d, \s1\().2d, \s1\().2d + sri vtmp.2d, \s1\().2d, #63 + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro xar_m1 d, s0, s1, imm + eor vtmp.16b, \s0\().16b, \s1\().16b + shl \d\().2d, vtmp.2d, #(64-\imm) + sri \d\().2d, vtmp.2d, #(\imm) +.endm + +.macro bcax_m1 d, s0, s1, s2 + bic vtmp.16b, \s1\().16b, \s2\().16b + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) +#define STACK_OFFSET_OUTER (4*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// NOTE: This is copied from the de-symbolized version of +// keccak_f1600_x1_scalar_symbolic.S. When changing the scalar code, +// SLOTHY needs to be re-run to re-allocate registers and stack spills, +// and the resulting code copied here. +// +// TODO: Automate this +.macro scalar_round_initial + eor x30, x24, x25 // *..................................................... + eor x27, x9, x10 // *..................................................... + eor x0, x30, x21 // .*.................................................... + eor x26, x27, x6 // .*.................................................... + eor x27, x26, x7 // ..*................................................... + eor x29, x0, x22 // ..*................................................... + eor x26, x29, x23 // ...*.................................................. + eor x29, x4, x5 // ...*.................................................. + eor x30, x29, x1 // ....*................................................. + eor x0, x27, x8 // ....*................................................. + eor x29, x30, x2 // .....*................................................ + eor x30, x19, x20 // .....*................................................ + eor x30, x30, x16 // ......*............................................... + eor x27, x26, x0, ror #63 // ......*............................................... + eor x4, x4, x27 // .......*.............................................. + eor x30, x30, x17 // .......*.............................................. + eor x30, x30, x28 // ........*............................................. + eor x29, x29, x3 // ........*............................................. + eor x0, x0, x30, ror #63 // .........*............................................ + eor x30, x30, x29, ror #63 // .........*............................................ + eor x22, x22, x30 // ..........*........................................... + eor x23, x23, x30 // ..........*........................................... + str x23, [sp, #STACK_LOC_0] // ...........*.......................................... // @slothy:writes=stack_0 + eor x23, x14, x15 // ...........*.......................................... + eor x14, x14, x0 // ............*......................................... + eor x23, x23, x11 // ............*......................................... + eor x15, x15, x0 // .............*........................................ + eor x1, x1, x27 // .............*........................................ + eor x23, x23, x12 // ..............*....................................... + eor x23, x23, x13 // ...............*...................................... + eor x11, x11, x0 // ...............*...................................... + eor x29, x29, x23, ror #63 // ................*..................................... + eor x23, x23, x26, ror #63 // ................*..................................... + eor x26, x13, x0 // .................*.................................... + eor x13, x28, x23 // .................*.................................... + eor x28, x24, x30 // ..................*................................... + eor x24, x16, x23 // ..................*................................... + eor x16, x21, x30 // ...................*.................................. + eor x21, x25, x30 // ...................*.................................. + eor x30, x19, x23 // ....................*................................. + eor x19, x20, x23 // ....................*................................. + eor x20, x17, x23 // .....................*................................ + eor x17, x12, x0 // .....................*................................ + eor x0, x2, x27 // ......................*............................... + eor x2, x6, x29 // ......................*............................... + eor x6, x8, x29 // .......................*.............................. + bic x8, x28, x13, ror #47 // .......................*.............................. + eor x12, x3, x27 // ........................*............................. + bic x3, x13, x17, ror #19 // ........................*............................. + eor x5, x5, x27 // .........................*............................ + ldr x27, [sp, #STACK_LOC_0] // .........................*............................ // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 // ..........................*........................... + eor x9, x9, x29 // ..........................*........................... + eor x23, x25, x5, ror #52 // ...........................*.......................... + eor x3, x3, x2, ror #24 // ...........................*.......................... + eor x8, x8, x17, ror #2 // ............................*......................... + eor x17, x10, x29 // ............................*......................... + bic x25, x12, x22, ror #47 // .............................*........................ + eor x29, x7, x29 // .............................*........................ + bic x10, x4, x27, ror #2 // ..............................*....................... + bic x7, x5, x28, ror #10 // ..............................*....................... + eor x10, x10, x20, ror #50 // ...............................*...................... + eor x13, x7, x13, ror #57 // ...............................*...................... + bic x7, x2, x5, ror #47 // ................................*..................... + eor x2, x25, x24, ror #39 // ................................*..................... + bic x25, x20, x11, ror #57 // .................................*.................... + bic x5, x17, x4, ror #25 // .................................*.................... + eor x25, x25, x17, ror #53 // ..................................*................... + bic x17, x11, x17, ror #60 // ..................................*................... + eor x28, x7, x28, ror #57 // ...................................*.................. + bic x7, x9, x12, ror #42 // ...................................*.................. + eor x7, x7, x22, ror #25 // ....................................*................. + bic x22, x22, x24, ror #56 // ....................................*................. + bic x24, x24, x15, ror #31 // .....................................*................ + eor x22, x22, x15, ror #23 // .....................................*................ + bic x20, x27, x20, ror #48 // ......................................*............... + bic x15, x15, x9, ror #16 // ......................................*............... + eor x12, x15, x12, ror #58 // .......................................*.............. + eor x15, x5, x27, ror #27 // .......................................*.............. + eor x5, x20, x11, ror #41 // ........................................*............. + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ........................................*............. + eor x20, x17, x4, ror #21 // .........................................*............ + eor x17, x24, x9, ror #47 // .........................................*............ + mov x24, #1 // ..........................................*........... + bic x9, x0, x16, ror #9 // ..........................................*........... + str x24, [sp, #STACK_OFFSET_COUNT] // ...........................................*.......... // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 // ...........................................*.......... + bic x27, x1, x21, ror #50 // ............................................*......... + bic x4, x26, x29, ror #63 // ............................................*......... + eor x1, x1, x4, ror #21 // .............................................*........ + ldr x11, [x11] // .............................................*........ + bic x4, x21, x30, ror #57 // ..............................................*....... + eor x21, x24, x21, ror #30 // ..............................................*....... + eor x24, x9, x19, ror #44 // ...............................................*...... + bic x9, x14, x6, ror #5 // ...............................................*...... + eor x9, x9, x0, ror #43 // ................................................*..... + bic x0, x6, x0, ror #38 // ................................................*..... + eor x1, x1, x11 // .................................................*.... + eor x11, x4, x26, ror #35 // .................................................*.... + eor x4, x0, x16, ror #47 // ..................................................*... + bic x0, x16, x19, ror #35 // ..................................................*... + eor x16, x27, x30, ror #43 // ...................................................*.. + bic x27, x30, x26, ror #42 // ...................................................*.. + bic x26, x19, x14, ror #41 // ....................................................*. + eor x19, x0, x14, ror #12 // ....................................................*. + eor x14, x26, x6, ror #46 // .....................................................* + eor x6, x27, x29, ror #41 // .....................................................* + + // eor5 X, sAma, sAsa, tAba, sAga, sAka + // eor5 X, sAme, sAse, sAbe, sAge, sAke + // eor5 X, sAmi, sAsi, sAbi, sAgi, sAki + // eor5 X, sAmo, sAso, sAbo, sAgo, sAko + // eor5 X, sAmu, sAsu, sAbu, sAgu, sAku + + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, tAba, X + // eor X, sAbi, X + // eor X, sAki, X + // eor X, sAko, X + // eor X, sAmu, X + // eor X, sAso, X + // eor X, sAma, X + // eor X, sAbe, X + // eor X, sAgo, X + // eor X, sAme, X + // eor X, sAgi, X + // eor X, sAka, X + // eor X, sAbo, X + // eor X, sAmo, X + // eor X, sAmi, X + // eor X, sAke, X + // eor X, sAgu, X + // eor X, sAsi, X + // eor X, sAku, X + // eor X, sAsa, X + // eor X, sAbu, X + // eor X, sAsu, X + // eor X, sAse, X + // eor X, sAga, X + // eor X, sAge, X + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [X] + // mov X, #1 + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro vector_round + eor3_m1 C0, vAba, vAga, vAka + eor3_m1 C0, C0, vAma, vAsa + eor3_m1 C1, vAbe, vAge, vAke + eor3_m1 C1, C1, vAme, vAse + eor3_m1 C2, vAbi, vAgi, vAki + eor3_m1 C2, C2, vAmi, vAsi + eor3_m1 C3, vAbo, vAgo, vAko + eor3_m1 C3, C3, vAmo, vAso + eor3_m1 C4, vAbu, vAgu, vAku + eor3_m1 C4, C4, vAmu, vAsu + rax1_m1 E1, C0, C2 + rax1_m1 E3, C2, C4 + rax1_m1 E0, C4, C1 + rax1_m1 E2, C1, C3 + rax1_m1 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m1 vAsa_, vAbi, E2, 2 + xar_m1 vAbi_, vAki, E2, 21 + xar_m1 vAki_, vAko, E3, 39 + xar_m1 vAko_, vAmu, E4, 56 + xar_m1 vAmu_, vAso, E3, 8 + xar_m1 vAso_, vAma, E0, 23 + xar_m1 vAka_, vAbe, E1, 63 + xar_m1 vAse_, vAgo, E3, 9 + xar_m1 vAgo_, vAme, E1, 19 + xar_m1 vAke_, vAgi, E2, 58 + xar_m1 vAgi_, vAka, E0, 61 + xar_m1 vAga_, vAbo, E3, 36 + xar_m1 vAbo_, vAmo, E3, 43 + xar_m1 vAmo_, vAmi, E2, 49 + xar_m1 vAmi_, vAke, E1, 54 + xar_m1 vAge_, vAgu, E4, 44 + xar_m1 vAgu_, vAsi, E2, 3 + xar_m1 vAsi_, vAku, E4, 25 + xar_m1 vAku_, vAsa, E0, 46 + xar_m1 vAma_, vAbu, E4, 37 + xar_m1 vAbu_, vAsu, E4, 50 + xar_m1 vAsu_, vAse, E1, 62 + xar_m1 vAme_, vAga, E0, 28 + xar_m1 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m1 vAga, vAga_, vAgi_, vAge_ + bcax_m1 vAge, vAge_, vAgo_, vAgi_ + bcax_m1 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m1 vAgo, vAgo_, vAga_, vAgu_ + bcax_m1 vAgu, vAgu_, vAge_, vAga_ + bcax_m1 vAka, vAka_, vAki_, vAke_ + bcax_m1 vAke, vAke_, vAko_, vAki_ + bcax_m1 vAki, vAki_, vAku_, vAko_ + bcax_m1 vAko, vAko_, vAka_, vAku_ + bcax_m1 vAku, vAku_, vAke_, vAka_ + bcax_m1 vAma, vAma_, vAmi_, vAme_ + bcax_m1 vAme, vAme_, vAmo_, vAmi_ + bcax_m1 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m1 vAmo, vAmo_, vAma_, vAmu_ + bcax_m1 vAmu, vAmu_, vAme_, vAma_ + bcax_m1 vAsa, vAsa_, vAsi_, vAse_ + bcax_m1 vAse, vAse_, vAso_, vAsi_ + bcax_m1 vAsi, vAsi_, vAsu_, vAso_ + bcax_m1 vAso, vAso_, vAsa_, vAsu_ + bcax_m1 vAsu, vAsu_, vAse_, vAsa_ + bcax_m1 vAba, vAba_, vAbi_, vAbe_ + bcax_m1 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m1 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m1 vAbo, vAbo_, vAba_, vAbu_ + bcax_m1 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 // *........................................................ + eor x0, x0, x13, ror #48 // .*....................................................... + eor x26, x8, x9, ror #57 // .*....................................................... + eor x27, x0, x14, ror #10 // ..*...................................................... + eor x29, x16, x28, ror #63 // ..*...................................................... + eor x26, x26, x6, ror #51 // ...*..................................................... + eor x30, x23, x22, ror #50 // ...*..................................................... + eor x0, x26, x10, ror #31 // ....*.................................................... + eor x29, x29, x19, ror #37 // ....*.................................................... + eor x27, x27, x12, ror #5 // .....*................................................... + eor x30, x30, x24, ror #34 // .....*................................................... + eor x0, x0, x7, ror #27 // ......*.................................................. + eor x26, x30, x21, ror #26 // ......*.................................................. + eor x26, x26, x25, ror #15 // .......*................................................. + ror x30, x27, #62 // .......*................................................. + eor x30, x30, x26, ror #57 // ........*................................................ + ror x26, x26, #58 // ........*................................................ + eor x16, x30, x16 // .........*............................................... + eor x28, x30, x28, ror #63 // .........*............................................... + str x28, [sp, #STACK_LOC_0] // ..........*.............................................. // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 // ..........*.............................................. + eor x28, x1, x2, ror #61 // ...........*............................................. + eor x19, x30, x19, ror #37 // ...........*............................................. + eor x29, x29, x20, ror #2 // ............*............................................ + eor x28, x28, x4, ror #54 // ............*............................................ + eor x26, x26, x0, ror #55 // .............*........................................... + eor x28, x28, x3, ror #39 // .............*........................................... + eor x28, x28, x5, ror #25 // ..............*.......................................... + ror x0, x0, #56 // ..............*.......................................... + eor x0, x0, x29, ror #63 // ...............*......................................... + eor x27, x28, x27, ror #61 // ...............*......................................... + eor x13, x0, x13, ror #46 // ................*........................................ + eor x28, x29, x28, ror #63 // ................*........................................ + eor x29, x30, x20, ror #2 // .................*....................................... + eor x20, x26, x3, ror #39 // .................*....................................... + eor x11, x0, x11, ror #50 // ..................*...................................... + eor x25, x28, x25, ror #9 // ..................*...................................... + eor x3, x28, x21, ror #20 // ...................*..................................... + eor x21, x26, x1 // ...................*..................................... + eor x9, x27, x9, ror #49 // ....................*.................................... + eor x24, x28, x24, ror #28 // ....................*.................................... + eor x1, x30, x17, ror #36 // .....................*................................... + eor x14, x0, x14, ror #8 // .....................*................................... + eor x22, x28, x22, ror #44 // ......................*.................................. + eor x8, x27, x8, ror #56 // ......................*.................................. + eor x17, x27, x7, ror #19 // .......................*................................. + eor x15, x0, x15, ror #62 // .......................*................................. + bic x7, x20, x22, ror #47 // ........................*................................ + eor x4, x26, x4, ror #54 // ........................*................................ + eor x0, x0, x12, ror #3 // .........................*............................... + eor x28, x28, x23, ror #58 // .........................*............................... + eor x23, x26, x2, ror #61 // ..........................*.............................. + eor x26, x26, x5, ror #25 // ..........................*.............................. + eor x2, x7, x16, ror #39 // ...........................*............................. + bic x7, x9, x20, ror #42 // ...........................*............................. + bic x30, x15, x9, ror #16 // ............................*............................ + eor x7, x7, x22, ror #25 // ............................*............................ + eor x12, x30, x20, ror #58 // .............................*........................... + bic x20, x22, x16, ror #56 // .............................*........................... + eor x30, x27, x6, ror #43 // ..............................*.......................... + eor x22, x20, x15, ror #23 // ..............................*.......................... + bic x6, x19, x13, ror #42 // ...............................*......................... + eor x6, x6, x17, ror #41 // ................................*........................ + bic x5, x13, x17, ror #63 // ................................*........................ + eor x5, x21, x5, ror #21 // .................................*....................... + bic x17, x17, x21, ror #44 // .................................*....................... + eor x27, x27, x10, ror #23 // ..................................*...................... + bic x21, x21, x25, ror #50 // ..................................*...................... + bic x20, x27, x4, ror #25 // ...................................*..................... + bic x10, x16, x15, ror #31 // ...................................*..................... + eor x16, x21, x19, ror #43 // ....................................*.................... + eor x21, x17, x25, ror #30 // ....................................*.................... + bic x19, x25, x19, ror #57 // .....................................*................... + ldr x25, [sp, #STACK_OFFSET_COUNT] // .....................................*................... // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 // ......................................*.................. + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // ......................................*.................. + eor x15, x20, x28, ror #27 // .......................................*................. + bic x20, x4, x28, ror #2 // .......................................*................. + eor x10, x20, x1, ror #50 // ........................................*................ + bic x20, x11, x27, ror #60 // ........................................*................ + eor x20, x20, x4, ror #21 // .........................................*............... + bic x4, x28, x1, ror #48 // .........................................*............... + bic x1, x1, x11, ror #57 // ..........................................*.............. + ldr x28, [x9, x25, LSL #3] // ..........................................*.............. + ldr x9, [sp, #STACK_LOC_0] // ...........................................*............. // @slothy:reads=stack_0 + add x25, x25, #1 // ...........................................*............. + str x25, [sp, #STACK_OFFSET_COUNT] // ............................................*............ // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ............................................*............ // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 // .............................................*........... + bic x27, x30, x26, ror #47 // .............................................*........... + eor x1, x5, x28 // ..............................................*.......... + eor x5, x4, x11, ror #41 // ..............................................*.......... + eor x11, x19, x13, ror #35 // ...............................................*......... + bic x13, x26, x24, ror #10 // ...............................................*......... + eor x28, x27, x24, ror #57 // ................................................*........ + bic x27, x24, x9, ror #47 // ................................................*........ + bic x19, x23, x3, ror #9 // .................................................*....... + bic x4, x29, x14, ror #41 // .................................................*....... + eor x24, x19, x29, ror #44 // ..................................................*...... + bic x29, x3, x29, ror #35 // ..................................................*...... + eor x13, x13, x9, ror #57 // ...................................................*..... + eor x19, x29, x14, ror #12 // ...................................................*..... + bic x29, x9, x0, ror #19 // ....................................................*.... + bic x14, x14, x8, ror #5 // ....................................................*.... + eor x9, x14, x23, ror #43 // .....................................................*... + eor x14, x4, x8, ror #46 // .....................................................*... + bic x23, x8, x23, ror #38 // ......................................................*.. + eor x8, x27, x0, ror #2 // ......................................................*.. + eor x4, x23, x3, ror #47 // .......................................................*. + bic x3, x0, x30, ror #5 // .......................................................*. + eor x23, x3, x26, ror #52 // ........................................................* + eor x3, x29, x30, ror #24 // ........................................................* + + // eor X, tAba, sAga, ror #61 + // eor X, X, sAma, ror #54 + // eor X, X, sAka, ror #39 + // eor X, X, sAsa, ror #25 + + // eor X, sAke, sAme, ror #57 + // eor X, X, sAbe, ror #51 + // eor X, X, sAse, ror #31 + // eor X, X, sAge, ror #27 + + // eor X, sAsi, sAbi, ror #52 + // eor X, X, sAki, ror #48 + // eor X, X, sAmi, ror #10 + // eor X, X, sAgi, ror #5 + + // eor X, sAbo, sAko, ror #63 + // eor X, X, sAmo, ror #37 + // eor X, X, sAgo, ror #36 + // eor X, X, sAso, ror #2 + + // eor X, sAku, sAgu, ror #50 + // eor X, X, sAmu, ror #34 + // eor X, X, sAbu, ror #26 + // eor X, X, sAsu, ror #15 + + // eor X, X, X, ror #61 + // ror X, X, #62 + // eor X, X, X, ror #57 + // ror X, X, #58 + // eor X, X, X, ror #55 + // ror X, X, #56 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, X, tAba + // eor X, X, sAbi, ror #50 + // eor X, X, sAki, ror #46 + // eor X, X, sAko, ror #63 + // eor X, X, sAmu, ror #28 + // eor X, X, sAso, ror #2 + // eor X, X, sAma, ror #54 + // eor X, X, sAbe, ror #43 + // eor X, X, sAgo, ror #36 + // eor X, X, sAme, ror #49 + // eor X, X, sAgi, ror #3 + // eor X, X, sAka, ror #39 + // eor X, X, sAbo + // eor X, X, sAmo, ror #37 + // eor X, X, sAmi, ror #8 + // eor X, X, sAke, ror #56 + // eor X, X, sAgu, ror #44 + // eor X, X, sAsi, ror #62 + // eor X, X, sAku, ror #58 + // eor X, X, sAsa, ror #25 + // eor X, X, sAbu, ror #20 + // eor X, X, sAsu, ror #9 + // eor X, X, sAse, ror #23 + // eor X, X, sAga, ror #61 + // eor X, X, sAge, ror #19 + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + // ldr X, [X, X, LSL #3] + // add X, X, #1 + // cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + mov outer, #0 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input + load_input_scalar 0 // First scalar input + +mld_keccak_f1600_x4_v8a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x4_v8a_scalar_hybrid_loop + final_scalar_rotate + + // Read outer loop flag: We repeat the above twice + ldr outer, [sp, #STACK_OFFSET_OUTER] // @slothy:reads=STACK_OFFSET_OUTER + cmp outer, #1 + beq mld_keccak_f1600_x4_v8a_scalar_hybrid_done + + // Update outer loop flag + mov outer, #1 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 // Store first scalar data + load_input_scalar 1 // Load second scalar input + + b mld_keccak_f1600_x4_v8a_scalar_hybrid_initial +mld_keccak_f1600_x4_v8a_scalar_hybrid_done: + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 1 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq outer + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq vtmp + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_OFFSET_OUTER +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..293e246828 --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean.S @@ -0,0 +1,1052 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations (clean version for optimization) + Signature: void mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t *state + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + outer .req x30 + cur_const .req x26 + + /* Mapping of Kecck-f1600 SIMD state to vector registers + * at the beginning and end of each round. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Unused temporary */ + vtmp .req v31 + + /* Mapping of Kecck-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro eor3_m1 d, s0, s1, s2 + eor \d\().16b, \s0\().16b, \s1\().16b + eor \d\().16b, \d\().16b, \s2\().16b +.endm + +.macro rax1_m1 d, s0, s1 + add vtmp.2d, \s1\().2d, \s1\().2d + sri vtmp.2d, \s1\().2d, #63 + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro xar_m1 d, s0, s1, imm + eor vtmp.16b, \s0\().16b, \s1\().16b + shl \d\().2d, vtmp.2d, #(64-\imm) + sri \d\().2d, vtmp.2d, #(\imm) +.endm + +.macro bcax_m1 d, s0, s1, s2 + bic vtmp.16b, \s1\().16b, \s2\().16b + eor \d\().16b, vtmp.16b, \s0\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) +#define STACK_OFFSET_OUTER (4*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// NOTE: This is copied from the de-symbolized version of +// keccak_f1600_x1_scalar_symbolic.S. When changing the scalar code, +// SLOTHY needs to be re-run to re-allocate registers and stack spills, +// and the resulting code copied here. +// +// TODO: Automate this +.macro scalar_round_initial + eor x30, x24, x25 // *..................................................... + eor x27, x9, x10 // *..................................................... + eor x0, x30, x21 // .*.................................................... + eor x26, x27, x6 // .*.................................................... + eor x27, x26, x7 // ..*................................................... + eor x29, x0, x22 // ..*................................................... + eor x26, x29, x23 // ...*.................................................. + eor x29, x4, x5 // ...*.................................................. + eor x30, x29, x1 // ....*................................................. + eor x0, x27, x8 // ....*................................................. + eor x29, x30, x2 // .....*................................................ + eor x30, x19, x20 // .....*................................................ + eor x30, x30, x16 // ......*............................................... + eor x27, x26, x0, ror #63 // ......*............................................... + eor x4, x4, x27 // .......*.............................................. + eor x30, x30, x17 // .......*.............................................. + eor x30, x30, x28 // ........*............................................. + eor x29, x29, x3 // ........*............................................. + eor x0, x0, x30, ror #63 // .........*............................................ + eor x30, x30, x29, ror #63 // .........*............................................ + eor x22, x22, x30 // ..........*........................................... + eor x23, x23, x30 // ..........*........................................... + str x23, [sp, #STACK_LOC_0] // ...........*.......................................... // @slothy:writes=stack_0 + eor x23, x14, x15 // ...........*.......................................... + eor x14, x14, x0 // ............*......................................... + eor x23, x23, x11 // ............*......................................... + eor x15, x15, x0 // .............*........................................ + eor x1, x1, x27 // .............*........................................ + eor x23, x23, x12 // ..............*....................................... + eor x23, x23, x13 // ...............*...................................... + eor x11, x11, x0 // ...............*...................................... + eor x29, x29, x23, ror #63 // ................*..................................... + eor x23, x23, x26, ror #63 // ................*..................................... + eor x26, x13, x0 // .................*.................................... + eor x13, x28, x23 // .................*.................................... + eor x28, x24, x30 // ..................*................................... + eor x24, x16, x23 // ..................*................................... + eor x16, x21, x30 // ...................*.................................. + eor x21, x25, x30 // ...................*.................................. + eor x30, x19, x23 // ....................*................................. + eor x19, x20, x23 // ....................*................................. + eor x20, x17, x23 // .....................*................................ + eor x17, x12, x0 // .....................*................................ + eor x0, x2, x27 // ......................*............................... + eor x2, x6, x29 // ......................*............................... + eor x6, x8, x29 // .......................*.............................. + bic x8, x28, x13, ror #47 // .......................*.............................. + eor x12, x3, x27 // ........................*............................. + bic x3, x13, x17, ror #19 // ........................*............................. + eor x5, x5, x27 // .........................*............................ + ldr x27, [sp, #STACK_LOC_0] // .........................*............................ // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 // ..........................*........................... + eor x9, x9, x29 // ..........................*........................... + eor x23, x25, x5, ror #52 // ...........................*.......................... + eor x3, x3, x2, ror #24 // ...........................*.......................... + eor x8, x8, x17, ror #2 // ............................*......................... + eor x17, x10, x29 // ............................*......................... + bic x25, x12, x22, ror #47 // .............................*........................ + eor x29, x7, x29 // .............................*........................ + bic x10, x4, x27, ror #2 // ..............................*....................... + bic x7, x5, x28, ror #10 // ..............................*....................... + eor x10, x10, x20, ror #50 // ...............................*...................... + eor x13, x7, x13, ror #57 // ...............................*...................... + bic x7, x2, x5, ror #47 // ................................*..................... + eor x2, x25, x24, ror #39 // ................................*..................... + bic x25, x20, x11, ror #57 // .................................*.................... + bic x5, x17, x4, ror #25 // .................................*.................... + eor x25, x25, x17, ror #53 // ..................................*................... + bic x17, x11, x17, ror #60 // ..................................*................... + eor x28, x7, x28, ror #57 // ...................................*.................. + bic x7, x9, x12, ror #42 // ...................................*.................. + eor x7, x7, x22, ror #25 // ....................................*................. + bic x22, x22, x24, ror #56 // ....................................*................. + bic x24, x24, x15, ror #31 // .....................................*................ + eor x22, x22, x15, ror #23 // .....................................*................ + bic x20, x27, x20, ror #48 // ......................................*............... + bic x15, x15, x9, ror #16 // ......................................*............... + eor x12, x15, x12, ror #58 // .......................................*.............. + eor x15, x5, x27, ror #27 // .......................................*.............. + eor x5, x20, x11, ror #41 // ........................................*............. + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ........................................*............. + eor x20, x17, x4, ror #21 // .........................................*............ + eor x17, x24, x9, ror #47 // .........................................*............ + mov x24, #1 // ..........................................*........... + bic x9, x0, x16, ror #9 // ..........................................*........... + str x24, [sp, #STACK_OFFSET_COUNT] // ...........................................*.......... // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 // ...........................................*.......... + bic x27, x1, x21, ror #50 // ............................................*......... + bic x4, x26, x29, ror #63 // ............................................*......... + eor x1, x1, x4, ror #21 // .............................................*........ + ldr x11, [x11] // .............................................*........ + bic x4, x21, x30, ror #57 // ..............................................*....... + eor x21, x24, x21, ror #30 // ..............................................*....... + eor x24, x9, x19, ror #44 // ...............................................*...... + bic x9, x14, x6, ror #5 // ...............................................*...... + eor x9, x9, x0, ror #43 // ................................................*..... + bic x0, x6, x0, ror #38 // ................................................*..... + eor x1, x1, x11 // .................................................*.... + eor x11, x4, x26, ror #35 // .................................................*.... + eor x4, x0, x16, ror #47 // ..................................................*... + bic x0, x16, x19, ror #35 // ..................................................*... + eor x16, x27, x30, ror #43 // ...................................................*.. + bic x27, x30, x26, ror #42 // ...................................................*.. + bic x26, x19, x14, ror #41 // ....................................................*. + eor x19, x0, x14, ror #12 // ....................................................*. + eor x14, x26, x6, ror #46 // .....................................................* + eor x6, x27, x29, ror #41 // .....................................................* + + // eor5 X, sAma, sAsa, tAba, sAga, sAka + // eor5 X, sAme, sAse, sAbe, sAge, sAke + // eor5 X, sAmi, sAsi, sAbi, sAgi, sAki + // eor5 X, sAmo, sAso, sAbo, sAgo, sAko + // eor5 X, sAmu, sAsu, sAbu, sAgu, sAku + + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, tAba, X + // eor X, sAbi, X + // eor X, sAki, X + // eor X, sAko, X + // eor X, sAmu, X + // eor X, sAso, X + // eor X, sAma, X + // eor X, sAbe, X + // eor X, sAgo, X + // eor X, sAme, X + // eor X, sAgi, X + // eor X, sAka, X + // eor X, sAbo, X + // eor X, sAmo, X + // eor X, sAmi, X + // eor X, sAke, X + // eor X, sAgu, X + // eor X, sAsi, X + // eor X, sAku, X + // eor X, sAsa, X + // eor X, sAbu, X + // eor X, sAsu, X + // eor X, sAse, X + // eor X, sAga, X + // eor X, sAge, X + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [X] + // mov X, #1 + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m1 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m1 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m1 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m1 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m1 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m1 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m1 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m1 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m1 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m1 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m1 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m1 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m1 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m1 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m1 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m1 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m1 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m1 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m1 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m1 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m1 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m1 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m1 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m1 vAko, vAko_, vAka_, vAku_ + bcax_m1 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m1 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m1 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m1 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m1 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m1 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m1 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m1 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 // *........................................................ + eor x0, x0, x13, ror #48 // .*....................................................... + eor x26, x8, x9, ror #57 // .*....................................................... + eor x27, x0, x14, ror #10 // ..*...................................................... + eor x29, x16, x28, ror #63 // ..*...................................................... + eor x26, x26, x6, ror #51 // ...*..................................................... + eor x30, x23, x22, ror #50 // ...*..................................................... + eor x0, x26, x10, ror #31 // ....*.................................................... + eor x29, x29, x19, ror #37 // ....*.................................................... + eor x27, x27, x12, ror #5 // .....*................................................... + eor x30, x30, x24, ror #34 // .....*................................................... + eor x0, x0, x7, ror #27 // ......*.................................................. + eor x26, x30, x21, ror #26 // ......*.................................................. + eor x26, x26, x25, ror #15 // .......*................................................. + ror x30, x27, #62 // .......*................................................. + eor x30, x30, x26, ror #57 // ........*................................................ + ror x26, x26, #58 // ........*................................................ + eor x16, x30, x16 // .........*............................................... + eor x28, x30, x28, ror #63 // .........*............................................... + str x28, [sp, #STACK_LOC_0] // ..........*.............................................. // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 // ..........*.............................................. + eor x28, x1, x2, ror #61 // ...........*............................................. + eor x19, x30, x19, ror #37 // ...........*............................................. + eor x29, x29, x20, ror #2 // ............*............................................ + eor x28, x28, x4, ror #54 // ............*............................................ + eor x26, x26, x0, ror #55 // .............*........................................... + eor x28, x28, x3, ror #39 // .............*........................................... + eor x28, x28, x5, ror #25 // ..............*.......................................... + ror x0, x0, #56 // ..............*.......................................... + eor x0, x0, x29, ror #63 // ...............*......................................... + eor x27, x28, x27, ror #61 // ...............*......................................... + eor x13, x0, x13, ror #46 // ................*........................................ + eor x28, x29, x28, ror #63 // ................*........................................ + eor x29, x30, x20, ror #2 // .................*....................................... + eor x20, x26, x3, ror #39 // .................*....................................... + eor x11, x0, x11, ror #50 // ..................*...................................... + eor x25, x28, x25, ror #9 // ..................*...................................... + eor x3, x28, x21, ror #20 // ...................*..................................... + eor x21, x26, x1 // ...................*..................................... + eor x9, x27, x9, ror #49 // ....................*.................................... + eor x24, x28, x24, ror #28 // ....................*.................................... + eor x1, x30, x17, ror #36 // .....................*................................... + eor x14, x0, x14, ror #8 // .....................*................................... + eor x22, x28, x22, ror #44 // ......................*.................................. + eor x8, x27, x8, ror #56 // ......................*.................................. + eor x17, x27, x7, ror #19 // .......................*................................. + eor x15, x0, x15, ror #62 // .......................*................................. + bic x7, x20, x22, ror #47 // ........................*................................ + eor x4, x26, x4, ror #54 // ........................*................................ + eor x0, x0, x12, ror #3 // .........................*............................... + eor x28, x28, x23, ror #58 // .........................*............................... + eor x23, x26, x2, ror #61 // ..........................*.............................. + eor x26, x26, x5, ror #25 // ..........................*.............................. + eor x2, x7, x16, ror #39 // ...........................*............................. + bic x7, x9, x20, ror #42 // ...........................*............................. + bic x30, x15, x9, ror #16 // ............................*............................ + eor x7, x7, x22, ror #25 // ............................*............................ + eor x12, x30, x20, ror #58 // .............................*........................... + bic x20, x22, x16, ror #56 // .............................*........................... + eor x30, x27, x6, ror #43 // ..............................*.......................... + eor x22, x20, x15, ror #23 // ..............................*.......................... + bic x6, x19, x13, ror #42 // ...............................*......................... + eor x6, x6, x17, ror #41 // ................................*........................ + bic x5, x13, x17, ror #63 // ................................*........................ + eor x5, x21, x5, ror #21 // .................................*....................... + bic x17, x17, x21, ror #44 // .................................*....................... + eor x27, x27, x10, ror #23 // ..................................*...................... + bic x21, x21, x25, ror #50 // ..................................*...................... + bic x20, x27, x4, ror #25 // ...................................*..................... + bic x10, x16, x15, ror #31 // ...................................*..................... + eor x16, x21, x19, ror #43 // ....................................*.................... + eor x21, x17, x25, ror #30 // ....................................*.................... + bic x19, x25, x19, ror #57 // .....................................*................... + ldr x25, [sp, #STACK_OFFSET_COUNT] // .....................................*................... // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 // ......................................*.................. + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // ......................................*.................. + eor x15, x20, x28, ror #27 // .......................................*................. + bic x20, x4, x28, ror #2 // .......................................*................. + eor x10, x20, x1, ror #50 // ........................................*................ + bic x20, x11, x27, ror #60 // ........................................*................ + eor x20, x20, x4, ror #21 // .........................................*............... + bic x4, x28, x1, ror #48 // .........................................*............... + bic x1, x1, x11, ror #57 // ..........................................*.............. + ldr x28, [x9, x25, LSL #3] // ..........................................*.............. + ldr x9, [sp, #STACK_LOC_0] // ...........................................*............. // @slothy:reads=stack_0 + add x25, x25, #1 // ...........................................*............. + str x25, [sp, #STACK_OFFSET_COUNT] // ............................................*............ // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ............................................*............ // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 // .............................................*........... + bic x27, x30, x26, ror #47 // .............................................*........... + eor x1, x5, x28 // ..............................................*.......... + eor x5, x4, x11, ror #41 // ..............................................*.......... + eor x11, x19, x13, ror #35 // ...............................................*......... + bic x13, x26, x24, ror #10 // ...............................................*......... + eor x28, x27, x24, ror #57 // ................................................*........ + bic x27, x24, x9, ror #47 // ................................................*........ + bic x19, x23, x3, ror #9 // .................................................*....... + bic x4, x29, x14, ror #41 // .................................................*....... + eor x24, x19, x29, ror #44 // ..................................................*...... + bic x29, x3, x29, ror #35 // ..................................................*...... + eor x13, x13, x9, ror #57 // ...................................................*..... + eor x19, x29, x14, ror #12 // ...................................................*..... + bic x29, x9, x0, ror #19 // ....................................................*.... + bic x14, x14, x8, ror #5 // ....................................................*.... + eor x9, x14, x23, ror #43 // .....................................................*... + eor x14, x4, x8, ror #46 // .....................................................*... + bic x23, x8, x23, ror #38 // ......................................................*.. + eor x8, x27, x0, ror #2 // ......................................................*.. + eor x4, x23, x3, ror #47 // .......................................................*. + bic x3, x0, x30, ror #5 // .......................................................*. + eor x23, x3, x26, ror #52 // ........................................................* + eor x3, x29, x30, ror #24 // ........................................................* + + // eor X, tAba, sAga, ror #61 + // eor X, X, sAma, ror #54 + // eor X, X, sAka, ror #39 + // eor X, X, sAsa, ror #25 + + // eor X, sAke, sAme, ror #57 + // eor X, X, sAbe, ror #51 + // eor X, X, sAse, ror #31 + // eor X, X, sAge, ror #27 + + // eor X, sAsi, sAbi, ror #52 + // eor X, X, sAki, ror #48 + // eor X, X, sAmi, ror #10 + // eor X, X, sAgi, ror #5 + + // eor X, sAbo, sAko, ror #63 + // eor X, X, sAmo, ror #37 + // eor X, X, sAgo, ror #36 + // eor X, X, sAso, ror #2 + + // eor X, sAku, sAgu, ror #50 + // eor X, X, sAmu, ror #34 + // eor X, X, sAbu, ror #26 + // eor X, X, sAsu, ror #15 + + // eor X, X, X, ror #61 + // ror X, X, #62 + // eor X, X, X, ror #57 + // ror X, X, #58 + // eor X, X, X, ror #55 + // ror X, X, #56 + // eor X, X, X, ror #63 + // eor X, X, X, ror #63 + + // eor X, X, tAba + // eor X, X, sAbi, ror #50 + // eor X, X, sAki, ror #46 + // eor X, X, sAko, ror #63 + // eor X, X, sAmu, ror #28 + // eor X, X, sAso, ror #2 + // eor X, X, sAma, ror #54 + // eor X, X, sAbe, ror #43 + // eor X, X, sAgo, ror #36 + // eor X, X, sAme, ror #49 + // eor X, X, sAgi, ror #3 + // eor X, X, sAka, ror #39 + // eor X, X, sAbo + // eor X, X, sAmo, ror #37 + // eor X, X, sAmi, ror #8 + // eor X, X, sAke, ror #56 + // eor X, X, sAgu, ror #44 + // eor X, X, sAsi, ror #62 + // eor X, X, sAku, ror #58 + // eor X, X, sAsa, ror #25 + // eor X, X, sAbu, ror #20 + // eor X, X, sAsu, ror #9 + // eor X, X, sAse, ror #23 + // eor X, X, sAga, ror #61 + // eor X, X, sAge, ror #19 + + // ldr X, [sp, #STACK_OFFSET_CONST_SCALAR] + // ldr X, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + // ldr X, [X, X, LSL #3] + // add X, X, #1 + // cmp X, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + // str X, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + + // chi_step_ror sAga, X, X, X, 47, 39 + // chi_step_ror sAge, X, X, X, 42, 25 + // chi_step_ror sAgi, X, X, X, 16, 58 + // chi_step_ror sAgo, X, X, X, 31, 47 + // chi_step_ror sAgu, X, X, X, 56, 23 + // chi_step_ror sAka, X, X, X, 19, 24 + // chi_step_ror sAke, X, X, X, 47, 2 + // chi_step_ror sAki, X, X, X, 10, 57 + // chi_step_ror sAko, X, X, X, 47, 57 + // chi_step_ror sAku, X, X, X, 5, 52 + // chi_step_ror sAma, X, X, X, 38, 47 + // chi_step_ror sAme, X, X, X, 5, 43 + // chi_step_ror sAmi, X, X, X, 41, 46 + // chi_step_ror sAmo, X, X, X, 35, 12 + // chi_step_ror sAmu, X, X, X, 9, 44 + // chi_step_ror sAsa, X, X, X, 48, 41 + // chi_step_ror sAse, X, X, X, 2, 50 + // chi_step_ror sAsi, X, X, X, 25, 27 + // chi_step_ror sAso, X, X, X, 60, 21 + // chi_step_ror sAsu, X, X, X, 57, 53 + // chi_step_ror2 tAba, X, X, X, 63, 21 + // chi_step_ror sAbe, X, X, X, 42, 41 + // chi_step_ror sAbi, X, X, X, 57, 35 + // chi_step_ror sAbo, X, X, X, 50, 43 + // chi_step_ror sAbu, X, X, X, 44, 30 + + // eor tAba, tAba, X +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + mov outer, #0 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input + load_input_scalar 0 // First scalar input + +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + final_scalar_rotate + + // Read outer loop flag: We repeat the above twice + ldr outer, [sp, #STACK_OFFSET_OUTER] // @slothy:reads=STACK_OFFSET_OUTER + cmp outer, #1 + beq mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done + + // Update outer loop flag + mov outer, #1 + str outer, [sp, #STACK_OFFSET_OUTER] // @slothy:writes=STACK_OFFSET_OUTER + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 // Store first scalar data + load_input_scalar 1 // Load second scalar input + + b mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial +mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done: + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 1 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq outer + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq vtmp + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_OFFSET_OUTER +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/scripts/autogen b/scripts/autogen index 6f783c32a8..6cc399fe33 100755 --- a/scripts/autogen +++ b/scripts/autogen @@ -3684,6 +3684,26 @@ def gen_slothy(funcs): print(f"Failed to run SLOTHY on {t}!") exit(1) + # Rename `name` field in YAML block. SLOTHY copies the YAML block of + # its input, so kernels generated from a symbolic/clean source carry + # that source's name (*_clean, *_symbolic). Only those are renamed; + # other kernels keep their name, which need not equal `func` + # (e.g. poly_pointwise_montgomery_aarch64_asm). + out_path = f"{base}/{t}" + out_name = func + with open(out_path, "r") as f: + content = f.read() + new_content = re.sub( + r"^(\s*Name:\s*)\S+_(?:clean|symbolic)\s*$", + lambda m: m.group(1) + out_name, + content, + count=1, + flags=re.MULTILINE, + ) + if new_content != content: + with open(out_path, "w") as f: + f.write(new_content) + class BibliographyEntry: def __init__(self, raw_dict): @@ -5195,6 +5215,9 @@ def _main(): "rej_uniform_aarch64_asm", "rej_uniform_eta2_aarch64_asm", "rej_uniform_eta4_aarch64_asm", + "keccak_f1600_x1_scalar_aarch64_asm", + "keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm", + "keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm", ] parser = argparse.ArgumentParser( From a7b795bb6c104ddcbbe3715195f14d14a86bc9da Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:14 +0000 Subject: [PATCH 2/8] FIPS202: Add 3-way batched Keccak-f1600 with an AArch64 hybrid backend Add an opt-in 3-way Keccak-f1600 (MLD_CONFIG_USE_X3) with a C fallback and an AArch64 kernel that runs two states on Neon with SHA3 and one on scalar registers. It targets cores that issue SHA3 on all SIMD pipes, such as Neoverse V3. Default builds are unchanged. Signed-off-by: Soumya Bhattacharya --- BIBLIOGRAPHY.md | 1 + dev/fips202/aarch64/auto.h | 12 +- dev/fips202/aarch64/src/Makefile | 46 +- .../aarch64/src/fips202_native_aarch64.h | 14 + ..._f1600_x3_v84a_scalar_hybrid_aarch64_asm.S | 1598 +++++++++++++++++ .../aarch64/src/keccakf1600_round_constants.c | 7 +- dev/fips202/aarch64/x3_v84a_scalar.h | 37 + ...keccak_f1600_x3_v84a_scalar_hybrid_clean.S | 860 +++++++++ .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + .../mldsa_native/mldsa_native_config.h | 23 + mldsa/mldsa_native.c | 32 +- mldsa/mldsa_native_asm.S | 30 + mldsa/mldsa_native_config.h | 23 + mldsa/src/fips202/fips202x3.c | 184 ++ mldsa/src/fips202/fips202x3.h | 124 ++ mldsa/src/fips202/keccakf1600.c | 89 + mldsa/src/fips202/keccakf1600.h | 55 + mldsa/src/fips202/native/aarch64/auto.h | 12 +- .../aarch64/src/fips202_native_aarch64.h | 14 + ..._f1600_x3_v84a_scalar_hybrid_aarch64_asm.S | 647 +++++++ .../aarch64/src/keccakf1600_round_constants.c | 7 +- .../fips202/native/aarch64/x3_v84a_scalar.h | 37 + mldsa/src/fips202/native/api.h | 15 + mldsa/src/symmetric.h | 27 + proofs/cbmc/mldsa_native_config_cbmc.h | 23 + scripts/autogen | 8 +- test/configs/break_pct_config.h | 23 + test/configs/custom_heap_alloc_config.h | 23 + test/configs/custom_memcpy_config.h | 23 + test/configs/custom_memset_config.h | 23 + .../custom_native_capability_config_0.h | 23 + .../custom_native_capability_config_1.h | 23 + ...stom_native_capability_config_CPUID_AVX2.h | 23 + ...native_capability_config_ID_AA64PFR1_EL1.h | 23 + test/configs/custom_randombytes_config.h | 23 + test/configs/custom_stdlib_config.h | 23 + test/configs/custom_zeroize_config.h | 23 + test/configs/low_signing_bound_config.h | 23 + test/configs/no_asm_config.h | 23 + test/configs/serial_fips202_config.h | 23 + test/configs/test_alloc_config.h | 23 + test/configs/test_sign_hook_config.h | 23 + 53 files changed, 4557 insertions(+), 12 deletions(-) create mode 100644 dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 dev/fips202/aarch64/x3_v84a_scalar.h create mode 100644 dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S create mode 100644 mldsa/src/fips202/fips202x3.c create mode 100644 mldsa/src/fips202/fips202x3.h create mode 100644 mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h diff --git a/BIBLIOGRAPHY.md b/BIBLIOGRAPHY.md index f397dff750..33762b9be1 100644 --- a/BIBLIOGRAPHY.md +++ b/BIBLIOGRAPHY.md @@ -153,6 +153,7 @@ source code and documentation. - [mldsa/mldsa_native_config.h](mldsa/mldsa_native_config.h) - [mldsa/src/ct.h](mldsa/src/ct.h) - [mldsa/src/fips202/fips202.c](mldsa/src/fips202/fips202.c) + - [mldsa/src/fips202/fips202x3.c](mldsa/src/fips202/fips202x3.c) - [mldsa/src/fips202/fips202x4.c](mldsa/src/fips202/fips202x4.c) - [mldsa/src/packing.h](mldsa/src/packing.h) - [mldsa/src/poly.c](mldsa/src/poly.c) diff --git a/dev/fips202/aarch64/auto.h b/dev/fips202/aarch64/auto.h index 98045b2dbe..3cfdcf2473 100644 --- a/dev/fips202/aarch64/auto.h +++ b/dev/fips202/aarch64/auto.h @@ -67,9 +67,17 @@ */ #if defined(__APPLE__) || defined(MLD_SYS_AARCH64_FAST_SHA3) #include "x2_v84a.h" -#else +#elif defined(MLD_CONFIG_USE_X3) +/* + * 3-way hybrid (two states on Neon with SHA3, one on scalar registers, single + * pass), selected with MLD_CONFIG_USE_X3. Intended for cores that execute the + * SHA3 instructions on all SIMD pipes (e.g. Neoverse V3). When + * MLD_CONFIG_USE_X3 is not set, the x4 hybrid below is used. + */ +#include "x3_v84a_scalar.h" +#else /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && MLD_CONFIG_USE_X3 */ #include "x4_v8a_v84a_scalar.h" -#endif +#endif /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && !MLD_CONFIG_USE_X3 */ #else /* __ARM_FEATURE_SHA3 */ diff --git a/dev/fips202/aarch64/src/Makefile b/dev/fips202/aarch64/src/Makefile index 697ec4ca3f..ea95b7c5db 100644 --- a/dev/fips202/aarch64/src/Makefile +++ b/dev/fips202/aarch64/src/Makefile @@ -102,9 +102,53 @@ keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/kec -s mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop \ -e mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end +# x3 hybrid (2 Neon + 1 scalar, single pass): interleaving only; register +# allocation and the instruction order within each stream are kept. +keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S: ../../aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $^ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c ignore_objective=True \ + -c constraints.functional_only=True \ + -c constraints.allow_reordering=False \ + -c constraints.allow_spills=True \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x3_v84a_scalar_hybrid_initial \ + -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + + slothy-cli Arm_AArch64 Arm_Cortex_A55 \ + $@ -o $@ \ + -c reserved_regs="[x18,sp]" \ + -c inputs_are_outputs \ + -c variable_size \ + -c constraints.stalls_first_attempt=64 \ + -c ignore_objective=True \ + -c constraints.functional_only=True \ + -c constraints.allow_reordering=False \ + -c constraints.allow_spills=True \ + -c split_heuristic \ + -c split_heuristic_repeat=0 \ + -c split_heuristic_preprocess_naive_interleaving \ + -c split_heuristic_preprocess_naive_interleaving_strategy="alternate" \ + -c split_heuristic_estimate_performance=False \ + -c absorb_spills=False \ + -c outputs="[hint_STACK_OFFSET_COUNT]" \ + -s mld_keccak_f1600_x3_v84a_scalar_hybrid_loop \ + -e mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end + ALL=keccak_f1600_x1_scalar_aarch64_asm.S \ keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S \ - keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S + keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S \ + keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S all: $(ALL) diff --git a/dev/fips202/aarch64/src/fips202_native_aarch64.h b/dev/fips202/aarch64/src/fips202_native_aarch64.h index 3939c2359c..410d70e10a 100644 --- a/dev/fips202/aarch64/src/fips202_native_aarch64.h +++ b/dev/fips202/aarch64/src/fips202_native_aarch64.h @@ -66,4 +66,18 @@ __contract__( assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) ); +/* + * 3-way hybrid Keccak-f1600: states 0 and 1 on Neon (Armv8.4-A SHA3), state 2 + * on scalar registers with lazy rotations, interleaved 1:1 in a single pass. + */ +#define mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm \ + MLD_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) +); + #endif /* !MLD_DEV_FIPS202_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..3724e589c1 --- /dev/null +++ b/dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,1598 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/* ***************** REGISTER ALLOCATIONS ****************** */ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + cur_const .req x26 + + /* Mapping of Keccak-f1600 SIMD state to vector registers + * at the beginning and end of each round. Identical to x4: + * two independent states are packed into the 2 lanes of each + * 128-bit V register. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Mapping of Keccak-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/* *********************** MACROS *************************** */ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +/* Vector states are state0 (at input_addr) and state1 (at input_addr+0xc8). + * Identical layout to x4. */ +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +/* The single scalar state is state2, at offset (2)*25*8 from input_addr. + * x3 only ever uses idx=0 (there is no second scalar pass). */ +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// Scalar round macros are IDENTICAL to the x4 kernel; they operate on the +// single scalar state held in GPRs. +.macro scalar_round_initial + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #1 + bic x9, x0, x16, ror #9 + str x24, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 +.endm + +// Neon round: uses the single-instruction Armv8.4-A SHA3 forms (eor3, rax1, +// xar, bcax; the "_m0" macros) throughout. Requires __ARM_FEATURE_SHA3 +// (guarded above). +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m0 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m0 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m0 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m0 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m0 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m0 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m0 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m0 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m0 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m0 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m0 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m0 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m0 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m0 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m0 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m0 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m0 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m0 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m0 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m0 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m0 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m0 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m0 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m0 vAko, vAko_, vAka_, vAku_ + bcax_m0 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m0 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m0 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m0 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m0 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m0 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m0 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m0 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #62 + eor x30, x30, x26, ror #57 + ror x26, x26, #58 + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #56 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, LSL #3] + ldr x9, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + add x25, x25, #1 + str x25, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input (state0, state1) + load_input_scalar 0 // Scalar input (state2) + + // Single pass, 1:1 interleave. + // Initial iteration: scalar_round_initial does round 0 of the scalar state + // and sets the round counter to 1; vector_round does round 0 of the two + // vector states. Then the loop runs rounds 1..23 (23 iterations), one + // scalar + one vector round each, for a total of 24 rounds on all three + // states. + mld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + // Instructions: 176 + // Expected cycles: 88 + // Expected IPC: 2.00 + + // ---------------------------------- cycle (expected) -----------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|------------ + eor x30, x24, x25 // *....................................................................................... // @slothy:interleaving_class=0 + eor x27, x9, x10 // *....................................................................................... // @slothy:interleaving_class=0 + eor3 v30.16B, v0.16B, v5.16B, v10.16B // .*...................................................................................... // @slothy:interleaving_class=1 + eor3 v30.16B, v30.16B, v15.16B, v20.16B // .*...................................................................................... // @slothy:interleaving_class=1 + eor x0, x30, x21 // ..*..................................................................................... // @slothy:interleaving_class=0 + eor x26, x27, x6 // ..*..................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v1.16B, v6.16B, v11.16B // ...*.................................................................................... // @slothy:interleaving_class=1 + eor x27, x26, x7 // ...*.................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v29.16B, v16.16B, v21.16B // ....*................................................................................... // @slothy:interleaving_class=1 + eor x29, x0, x22 // ....*................................................................................... // @slothy:interleaving_class=0 + eor x26, x29, x23 // .....*.................................................................................. // @slothy:interleaving_class=0 + eor3 v28.16B, v2.16B, v7.16B, v12.16B // .....*.................................................................................. // @slothy:interleaving_class=1 + eor x29, x4, x5 // ......*................................................................................. // @slothy:interleaving_class=0 + eor3 v28.16B, v28.16B, v17.16B, v22.16B // ......*................................................................................. // @slothy:interleaving_class=1 + eor x30, x29, x1 // .......*................................................................................ // @slothy:interleaving_class=0 + eor x0, x27, x8 // .......*................................................................................ // @slothy:interleaving_class=0 + eor3 v27.16B, v3.16B, v8.16B, v13.16B // ........*............................................................................... // @slothy:interleaving_class=1 + eor x29, x30, x2 // ........*............................................................................... // @slothy:interleaving_class=0 + eor3 v27.16B, v27.16B, v18.16B, v23.16B // .........*.............................................................................. // @slothy:interleaving_class=1 + eor x30, x19, x20 // .........*.............................................................................. // @slothy:interleaving_class=0 + eor x30, x30, x16 // ..........*............................................................................. // @slothy:interleaving_class=0 + eor3 v26.16B, v4.16B, v9.16B, v14.16B // ..........*............................................................................. // @slothy:interleaving_class=1 + eor x27, x26, x0, ror #63 // ...........*............................................................................ // @slothy:interleaving_class=0 + eor3 v26.16B, v26.16B, v19.16B, v24.16B // ...........*............................................................................ // @slothy:interleaving_class=1 + eor x4, x4, x27 // ............*........................................................................... // @slothy:interleaving_class=0 + eor x30, x30, x17 // ............*........................................................................... // @slothy:interleaving_class=0 + rax1 v25.2D, v30.2D, v28.2D // .............*.......................................................................... // @slothy:interleaving_class=1 + eor x30, x30, x28 // .............*.......................................................................... // @slothy:interleaving_class=0 + eor x29, x29, x3 // ..............*......................................................................... // @slothy:interleaving_class=0 + rax1 v28.2D, v28.2D, v26.2D // ..............*......................................................................... // @slothy:interleaving_class=1 + eor x0, x0, x30, ror #63 // ...............*........................................................................ // @slothy:interleaving_class=0 + rax1 v26.2D, v26.2D, v29.2D // ...............*........................................................................ // @slothy:interleaving_class=1 + eor x30, x30, x29, ror #63 // ................*....................................................................... // @slothy:interleaving_class=0 + eor x22, x22, x30 // ................*....................................................................... // @slothy:interleaving_class=0 + rax1 v29.2D, v29.2D, v27.2D // .................*...................................................................... // @slothy:interleaving_class=1 + eor x23, x23, x30 // .................*...................................................................... // @slothy:interleaving_class=0 + rax1 v27.2D, v27.2D, v30.2D // ..................*..................................................................... // @slothy:interleaving_class=1 + str x23, [sp, #STACK_LOC_0] // ..................*..................................................................... // @slothy:writes=stack_0 // @slothy:interleaving_class=0 + eor x23, x14, x15 // ...................*.................................................................... // @slothy:interleaving_class=0 + eor v30.16B, v0.16B, v26.16B // ...................*.................................................................... // @slothy:interleaving_class=1 + eor x14, x14, x0 // ....................*................................................................... // @slothy:interleaving_class=0 + xar v0.2D, v2.2D, v29.2D, #2 // ....................*................................................................... // @slothy:interleaving_class=1 + eor x23, x23, x11 // .....................*.................................................................. // @slothy:interleaving_class=0 + eor x15, x15, x0 // .....................*.................................................................. // @slothy:interleaving_class=0 + xar v2.2D, v12.2D, v29.2D, #21 // ......................*................................................................. // @slothy:interleaving_class=1 + eor x1, x1, x27 // ......................*................................................................. // @slothy:interleaving_class=0 + xar v12.2D, v13.2D, v28.2D, #39 // .......................*................................................................ // @slothy:interleaving_class=1 + eor x23, x23, x12 // .......................*................................................................ // @slothy:interleaving_class=0 + eor x23, x23, x13 // ........................*............................................................... // @slothy:interleaving_class=0 + xar v13.2D, v19.2D, v27.2D, #56 // ........................*............................................................... // @slothy:interleaving_class=1 + eor x11, x11, x0 // .........................*.............................................................. // @slothy:interleaving_class=0 + eor x29, x29, x23, ror #63 // .........................*.............................................................. // @slothy:interleaving_class=0 + xar v19.2D, v23.2D, v28.2D, #8 // ..........................*............................................................. // @slothy:interleaving_class=1 + eor x23, x23, x26, ror #63 // ..........................*............................................................. // @slothy:interleaving_class=0 + xar v23.2D, v15.2D, v26.2D, #23 // ...........................*............................................................ // @slothy:interleaving_class=1 + eor x26, x13, x0 // ...........................*............................................................ // @slothy:interleaving_class=0 + eor x13, x28, x23 // ............................*........................................................... // @slothy:interleaving_class=0 + xar v15.2D, v1.2D, v25.2D, #63 // ............................*........................................................... // @slothy:interleaving_class=1 + eor x28, x24, x30 // .............................*.......................................................... // @slothy:interleaving_class=0 + xar v1.2D, v8.2D, v28.2D, #9 // .............................*.......................................................... // @slothy:interleaving_class=1 + eor x24, x16, x23 // ..............................*......................................................... // @slothy:interleaving_class=0 + eor x16, x21, x30 // ..............................*......................................................... // @slothy:interleaving_class=0 + xar v8.2D, v16.2D, v25.2D, #19 // ...............................*........................................................ // @slothy:interleaving_class=1 + eor x21, x25, x30 // ...............................*........................................................ // @slothy:interleaving_class=0 + xar v16.2D, v7.2D, v29.2D, #58 // ................................*....................................................... // @slothy:interleaving_class=1 + eor x30, x19, x23 // ................................*....................................................... // @slothy:interleaving_class=0 + eor x19, x20, x23 // .................................*...................................................... // @slothy:interleaving_class=0 + xar v7.2D, v10.2D, v26.2D, #61 // .................................*...................................................... // @slothy:interleaving_class=1 + eor x20, x17, x23 // ..................................*..................................................... // @slothy:interleaving_class=0 + xar v10.2D, v3.2D, v28.2D, #36 // ..................................*..................................................... // @slothy:interleaving_class=1 + eor x17, x12, x0 // ...................................*.................................................... // @slothy:interleaving_class=0 + eor x0, x2, x27 // ...................................*.................................................... // @slothy:interleaving_class=0 + xar v3.2D, v18.2D, v28.2D, #43 // ....................................*................................................... // @slothy:interleaving_class=1 + eor x2, x6, x29 // ....................................*................................................... // @slothy:interleaving_class=0 + xar v18.2D, v17.2D, v29.2D, #49 // .....................................*.................................................. // @slothy:interleaving_class=1 + eor x6, x8, x29 // .....................................*.................................................. // @slothy:interleaving_class=0 + bic x8, x28, x13, ror #47 // ......................................*................................................. // @slothy:interleaving_class=0 + xar v17.2D, v11.2D, v25.2D, #54 // ......................................*................................................. // @slothy:interleaving_class=1 + eor x12, x3, x27 // .......................................*................................................ // @slothy:interleaving_class=0 + bic x3, x13, x17, ror #19 // .......................................*................................................ // @slothy:interleaving_class=0 + xar v11.2D, v9.2D, v27.2D, #44 // ........................................*............................................... // @slothy:interleaving_class=1 + eor x5, x5, x27 // ........................................*............................................... // @slothy:interleaving_class=0 + xar v9.2D, v22.2D, v29.2D, #3 // .........................................*.............................................. // @slothy:interleaving_class=1 + ldr x27, [sp, #STACK_LOC_0] // .........................................*.............................................. // @slothy:reads=stack_0 // @slothy:interleaving_class=0 + bic x25, x17, x2, ror #5 // ..........................................*............................................. // @slothy:interleaving_class=0 + xar v22.2D, v14.2D, v27.2D, #25 // ..........................................*............................................. // @slothy:interleaving_class=1 + eor x9, x9, x29 // ...........................................*............................................ // @slothy:interleaving_class=0 + xar v14.2D, v20.2D, v26.2D, #46 // ...........................................*............................................ // @slothy:interleaving_class=1 + eor x23, x25, x5, ror #52 // ............................................*........................................... // @slothy:interleaving_class=0 + eor x3, x3, x2, ror #24 // ............................................*........................................... // @slothy:interleaving_class=0 + xar v20.2D, v4.2D, v27.2D, #37 // .............................................*.......................................... // @slothy:interleaving_class=1 + eor x8, x8, x17, ror #2 // .............................................*.......................................... // @slothy:interleaving_class=0 + xar v4.2D, v24.2D, v27.2D, #50 // ..............................................*......................................... // @slothy:interleaving_class=1 + eor x17, x10, x29 // ..............................................*......................................... // @slothy:interleaving_class=0 + bic x25, x12, x22, ror #47 // ...............................................*........................................ // @slothy:interleaving_class=0 + xar v24.2D, v21.2D, v25.2D, #62 // ...............................................*........................................ // @slothy:interleaving_class=1 + eor x29, x7, x29 // ................................................*....................................... // @slothy:interleaving_class=0 + xar v21.2D, v5.2D, v26.2D, #28 // ................................................*....................................... // @slothy:interleaving_class=1 + bic x10, x4, x27, ror #2 // .................................................*...................................... // @slothy:interleaving_class=0 + bic x7, x5, x28, ror #10 // .................................................*...................................... // @slothy:interleaving_class=0 + xar v27.2D, v6.2D, v25.2D, #20 // ..................................................*..................................... // @slothy:interleaving_class=1 + eor x10, x10, x20, ror #50 // ..................................................*..................................... // @slothy:interleaving_class=0 + eor x13, x7, x13, ror #57 // ...................................................*.................................... // @slothy:interleaving_class=0 + bcax v5.16B, v10.16B, v7.16B, v11.16B // ...................................................*.................................... // @slothy:interleaving_class=1 + bic x7, x2, x5, ror #47 // ....................................................*................................... // @slothy:interleaving_class=0 + bcax v6.16B, v11.16B, v8.16B, v7.16B // ....................................................*................................... // @slothy:interleaving_class=1 + eor x2, x25, x24, ror #39 // .....................................................*.................................. // @slothy:interleaving_class=0 + bic x25, x20, x11, ror #57 // .....................................................*.................................. // @slothy:interleaving_class=0 + bcax v7.16B, v7.16B, v9.16B, v8.16B // ......................................................*................................. // @slothy:interleaving_class=1 + bic x5, x17, x4, ror #25 // ......................................................*................................. // @slothy:interleaving_class=0 + bcax v8.16B, v8.16B, v10.16B, v9.16B // .......................................................*................................ // @slothy:interleaving_class=1 + eor x25, x25, x17, ror #53 // .......................................................*................................ // @slothy:interleaving_class=0 + bic x17, x11, x17, ror #60 // ........................................................*............................... // @slothy:interleaving_class=0 + bcax v9.16B, v9.16B, v11.16B, v10.16B // ........................................................*............................... // @slothy:interleaving_class=1 + eor x28, x7, x28, ror #57 // .........................................................*.............................. // @slothy:interleaving_class=0 + bcax v10.16B, v15.16B, v12.16B, v16.16B // .........................................................*.............................. // @slothy:interleaving_class=1 + bic x7, x9, x12, ror #42 // ..........................................................*............................. // @slothy:interleaving_class=0 + eor x7, x7, x22, ror #25 // ..........................................................*............................. // @slothy:interleaving_class=0 + bcax v11.16B, v16.16B, v13.16B, v12.16B // ...........................................................*............................ // @slothy:interleaving_class=1 + bic x22, x22, x24, ror #56 // ...........................................................*............................ // @slothy:interleaving_class=0 + bcax v12.16B, v12.16B, v14.16B, v13.16B // ............................................................*........................... // @slothy:interleaving_class=1 + bic x24, x24, x15, ror #31 // ............................................................*........................... // @slothy:interleaving_class=0 + eor x22, x22, x15, ror #23 // .............................................................*.......................... // @slothy:interleaving_class=0 + bcax v13.16B, v13.16B, v15.16B, v14.16B // .............................................................*.......................... // @slothy:interleaving_class=1 + bic x20, x27, x20, ror #48 // ..............................................................*......................... // @slothy:interleaving_class=0 + bcax v14.16B, v14.16B, v16.16B, v15.16B // ..............................................................*......................... // @slothy:interleaving_class=1 + bic x15, x15, x9, ror #16 // ...............................................................*........................ // @slothy:interleaving_class=0 + eor x12, x15, x12, ror #58 // ...............................................................*........................ // @slothy:interleaving_class=0 + bcax v15.16B, v20.16B, v17.16B, v21.16B // ................................................................*....................... // @slothy:interleaving_class=1 + eor x15, x5, x27, ror #27 // ................................................................*....................... // @slothy:interleaving_class=0 + eor x5, x20, x11, ror #41 // .................................................................*...................... // @slothy:interleaving_class=0 + bcax v16.16B, v21.16B, v18.16B, v17.16B // .................................................................*...................... // @slothy:interleaving_class=1 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ..................................................................*..................... // @slothy:interleaving_class=0 + bcax v17.16B, v17.16B, v19.16B, v18.16B // ..................................................................*..................... // @slothy:interleaving_class=1 + eor x20, x17, x4, ror #21 // ...................................................................*.................... // @slothy:interleaving_class=0 + eor x17, x24, x9, ror #47 // ...................................................................*.................... // @slothy:interleaving_class=0 + bcax v18.16B, v18.16B, v20.16B, v19.16B // ....................................................................*................... // @slothy:interleaving_class=1 + mov x24, #1 // ....................................................................*................... // @slothy:interleaving_class=0 + bcax v19.16B, v19.16B, v21.16B, v20.16B // .....................................................................*.................. // @slothy:interleaving_class=1 + bic x9, x0, x16, ror #9 // .....................................................................*.................. // @slothy:interleaving_class=0 + str x24, [sp, #STACK_OFFSET_COUNT] // ......................................................................*................. // @slothy:writes=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v20.16B, v0.16B, v22.16B, v1.16B // ......................................................................*................. // @slothy:interleaving_class=1 + bic x24, x29, x1, ror #44 // .......................................................................*................ // @slothy:interleaving_class=0 + bcax v21.16B, v1.16B, v23.16B, v22.16B // .......................................................................*................ // @slothy:interleaving_class=1 + bic x27, x1, x21, ror #50 // ........................................................................*............... // @slothy:interleaving_class=0 + bic x4, x26, x29, ror #63 // ........................................................................*............... // @slothy:interleaving_class=0 + bcax v22.16B, v22.16B, v24.16B, v23.16B // .........................................................................*.............. // @slothy:interleaving_class=1 + eor x1, x1, x4, ror #21 // .........................................................................*.............. // @slothy:interleaving_class=0 + bcax v23.16B, v23.16B, v0.16B, v24.16B // ..........................................................................*............. // @slothy:interleaving_class=1 + ldr x11, [x11] // ..........................................................................*............. // @slothy:interleaving_class=0 + bic x4, x21, x30, ror #57 // ...........................................................................*............ // @slothy:interleaving_class=0 + bcax v24.16B, v24.16B, v1.16B, v0.16B // ...........................................................................*............ // @slothy:interleaving_class=1 + eor x21, x24, x21, ror #30 // ............................................................................*........... // @slothy:interleaving_class=0 + eor x24, x9, x19, ror #44 // ............................................................................*........... // @slothy:interleaving_class=0 + bcax v0.16B, v30.16B, v2.16B, v27.16B // .............................................................................*.......... // @slothy:interleaving_class=1 + bic x9, x14, x6, ror #5 // .............................................................................*.......... // @slothy:interleaving_class=0 + bcax v1.16B, v27.16B, v3.16B, v2.16B // ..............................................................................*......... // @slothy:interleaving_class=1 + eor x9, x9, x0, ror #43 // ..............................................................................*......... // @slothy:interleaving_class=0 + bic x0, x6, x0, ror #38 // ...............................................................................*........ // @slothy:interleaving_class=0 + bcax v2.16B, v2.16B, v4.16B, v3.16B // ...............................................................................*........ // @slothy:interleaving_class=1 + eor x1, x1, x11 // ................................................................................*....... // @slothy:interleaving_class=0 + bcax v3.16B, v3.16B, v30.16B, v4.16B // ................................................................................*....... // @slothy:interleaving_class=1 + eor x11, x4, x26, ror #35 // .................................................................................*...... // @slothy:interleaving_class=0 + eor x4, x0, x16, ror #47 // .................................................................................*...... // @slothy:interleaving_class=0 + bcax v4.16B, v4.16B, v27.16B, v30.16B // ..................................................................................*..... // @slothy:interleaving_class=1 + bic x0, x16, x19, ror #35 // ..................................................................................*..... // @slothy:interleaving_class=0 + eor x16, x27, x30, ror #43 // ...................................................................................*.... // @slothy:interleaving_class=0 + bic x27, x30, x26, ror #42 // ...................................................................................*.... // @slothy:interleaving_class=0 + ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ....................................................................................*... // @slothy:reads=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + ld1r {v28.2D}, [x30], #8 // ....................................................................................*... // @slothy:interleaving_class=1 + bic x26, x19, x14, ror #41 // .....................................................................................*.. // @slothy:interleaving_class=0 + str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .....................................................................................*.. // @slothy:writes=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + eor x19, x0, x14, ror #12 // ......................................................................................*. // @slothy:interleaving_class=0 + eor x14, x26, x6, ror #46 // ......................................................................................*. // @slothy:interleaving_class=0 + eor v0.16B, v0.16B, v28.16B // .......................................................................................* // @slothy:interleaving_class=1 + eor x6, x27, x29, ror #41 // .......................................................................................* // @slothy:interleaving_class=0 + + // ---------------------------------- cycle (expected) -----------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|------------ + // eor x30, x24, x25 // *....................................................................................... + // eor x27, x9, x10 // *....................................................................................... + // eor x0, x30, x21 // ..*..................................................................................... + // eor x26, x27, x6 // ..*..................................................................................... + // eor x27, x26, x7 // ...*.................................................................................... + // eor x29, x0, x22 // ....*................................................................................... + // eor x26, x29, x23 // .....*.................................................................................. + // eor x29, x4, x5 // ......*................................................................................. + // eor x30, x29, x1 // .......*................................................................................ + // eor x0, x27, x8 // .......*................................................................................ + // eor x29, x30, x2 // ........*............................................................................... + // eor x30, x19, x20 // .........*.............................................................................. + // eor x30, x30, x16 // ..........*............................................................................. + // eor x27, x26, x0, ror #63 // ...........*............................................................................ + // eor x4, x4, x27 // ............*........................................................................... + // eor x30, x30, x17 // ............*........................................................................... + // eor x30, x30, x28 // .............*.......................................................................... + // eor x29, x29, x3 // ..............*......................................................................... + // eor x0, x0, x30, ror #63 // ...............*........................................................................ + // eor x30, x30, x29, ror #63 // ................*....................................................................... + // eor x22, x22, x30 // ................*....................................................................... + // eor x23, x23, x30 // .................*...................................................................... + // str x23, [sp, #STACK_LOC_0] // ..................*..................................................................... + // eor x23, x14, x15 // ...................*.................................................................... + // eor x14, x14, x0 // ....................*................................................................... + // eor x23, x23, x11 // .....................*.................................................................. + // eor x15, x15, x0 // .....................*.................................................................. + // eor x1, x1, x27 // ......................*................................................................. + // eor x23, x23, x12 // .......................*................................................................ + // eor x23, x23, x13 // ........................*............................................................... + // eor x11, x11, x0 // .........................*.............................................................. + // eor x29, x29, x23, ror #63 // .........................*.............................................................. + // eor x23, x23, x26, ror #63 // ..........................*............................................................. + // eor x26, x13, x0 // ...........................*............................................................ + // eor x13, x28, x23 // ............................*........................................................... + // eor x28, x24, x30 // .............................*.......................................................... + // eor x24, x16, x23 // ..............................*......................................................... + // eor x16, x21, x30 // ..............................*......................................................... + // eor x21, x25, x30 // ...............................*........................................................ + // eor x30, x19, x23 // ................................*....................................................... + // eor x19, x20, x23 // .................................*...................................................... + // eor x20, x17, x23 // ..................................*..................................................... + // eor x17, x12, x0 // ...................................*.................................................... + // eor x0, x2, x27 // ...................................*.................................................... + // eor x2, x6, x29 // ....................................*................................................... + // eor x6, x8, x29 // .....................................*.................................................. + // bic x8, x28, x13, ror #47 // ......................................*................................................. + // eor x12, x3, x27 // .......................................*................................................ + // bic x3, x13, x17, ror #19 // .......................................*................................................ + // eor x5, x5, x27 // ........................................*............................................... + // ldr x27, [sp, #STACK_LOC_0] // .........................................*.............................................. + // bic x25, x17, x2, ror #5 // ..........................................*............................................. + // eor x9, x9, x29 // ...........................................*............................................ + // eor x23, x25, x5, ror #52 // ............................................*........................................... + // eor x3, x3, x2, ror #24 // ............................................*........................................... + // eor x8, x8, x17, ror #2 // .............................................*.......................................... + // eor x17, x10, x29 // ..............................................*......................................... + // bic x25, x12, x22, ror #47 // ...............................................*........................................ + // eor x29, x7, x29 // ................................................*....................................... + // bic x10, x4, x27, ror #2 // .................................................*...................................... + // bic x7, x5, x28, ror #10 // .................................................*...................................... + // eor x10, x10, x20, ror #50 // ..................................................*..................................... + // eor x13, x7, x13, ror #57 // ...................................................*.................................... + // bic x7, x2, x5, ror #47 // ....................................................*................................... + // eor x2, x25, x24, ror #39 // .....................................................*.................................. + // bic x25, x20, x11, ror #57 // .....................................................*.................................. + // bic x5, x17, x4, ror #25 // ......................................................*................................. + // eor x25, x25, x17, ror #53 // .......................................................*................................ + // bic x17, x11, x17, ror #60 // ........................................................*............................... + // eor x28, x7, x28, ror #57 // .........................................................*.............................. + // bic x7, x9, x12, ror #42 // ..........................................................*............................. + // eor x7, x7, x22, ror #25 // ..........................................................*............................. + // bic x22, x22, x24, ror #56 // ...........................................................*............................ + // bic x24, x24, x15, ror #31 // ............................................................*........................... + // eor x22, x22, x15, ror #23 // .............................................................*.......................... + // bic x20, x27, x20, ror #48 // ..............................................................*......................... + // bic x15, x15, x9, ror #16 // ...............................................................*........................ + // eor x12, x15, x12, ror #58 // ...............................................................*........................ + // eor x15, x5, x27, ror #27 // ................................................................*....................... + // eor x5, x20, x11, ror #41 // .................................................................*...................... + // ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] // ..................................................................*..................... + // eor x20, x17, x4, ror #21 // ...................................................................*.................... + // eor x17, x24, x9, ror #47 // ...................................................................*.................... + // mov x24, #1 // ....................................................................*................... + // bic x9, x0, x16, ror #9 // .....................................................................*.................. + // str x24, [sp, #STACK_OFFSET_COUNT] // ......................................................................*................. + // bic x24, x29, x1, ror #44 // .......................................................................*................ + // bic x27, x1, x21, ror #50 // ........................................................................*............... + // bic x4, x26, x29, ror #63 // ........................................................................*............... + // eor x1, x1, x4, ror #21 // .........................................................................*.............. + // ldr x11, [x11] // ..........................................................................*............. + // bic x4, x21, x30, ror #57 // ...........................................................................*............ + // eor x21, x24, x21, ror #30 // ............................................................................*........... + // eor x24, x9, x19, ror #44 // ............................................................................*........... + // bic x9, x14, x6, ror #5 // .............................................................................*.......... + // eor x9, x9, x0, ror #43 // ..............................................................................*......... + // bic x0, x6, x0, ror #38 // ...............................................................................*........ + // eor x1, x1, x11 // ................................................................................*....... + // eor x11, x4, x26, ror #35 // .................................................................................*...... + // eor x4, x0, x16, ror #47 // .................................................................................*...... + // bic x0, x16, x19, ror #35 // ..................................................................................*..... + // eor x16, x27, x30, ror #43 // ...................................................................................*.... + // bic x27, x30, x26, ror #42 // ...................................................................................*.... + // bic x26, x19, x14, ror #41 // .....................................................................................*.. + // eor x19, x0, x14, ror #12 // ......................................................................................*. + // eor x14, x26, x6, ror #46 // ......................................................................................*. + // eor x6, x27, x29, ror #41 // .......................................................................................* + // eor3 v30.16b, v0.16b, v5.16b, v10.16b // .*...................................................................................... + // eor3 v30.16b, v30.16b, v15.16b, v20.16b // .*...................................................................................... + // eor3 v29.16b, v1.16b, v6.16b, v11.16b // ...*.................................................................................... + // eor3 v29.16b, v29.16b, v16.16b, v21.16b // ....*................................................................................... + // eor3 v28.16b, v2.16b, v7.16b, v12.16b // .....*.................................................................................. + // eor3 v28.16b, v28.16b, v17.16b, v22.16b // ......*................................................................................. + // eor3 v27.16b, v3.16b, v8.16b, v13.16b // ........*............................................................................... + // eor3 v27.16b, v27.16b, v18.16b, v23.16b // .........*.............................................................................. + // eor3 v26.16b, v4.16b, v9.16b, v14.16b // ..........*............................................................................. + // eor3 v26.16b, v26.16b, v19.16b, v24.16b // ...........*............................................................................ + // rax1 v25.2d, v30.2d, v28.2d // .............*.......................................................................... + // rax1 v28.2d, v28.2d, v26.2d // ..............*......................................................................... + // rax1 v26.2d, v26.2d, v29.2d // ...............*........................................................................ + // rax1 v29.2d, v29.2d, v27.2d // .................*...................................................................... + // rax1 v27.2d, v27.2d, v30.2d // ..................*..................................................................... + // eor v30.16b, v0.16b, v26.16b // ...................*.................................................................... + // xar v0.2d, v2.2d, v29.2d, #2 // ....................*................................................................... + // xar v2.2d, v12.2d, v29.2d, #21 // ......................*................................................................. + // xar v12.2d, v13.2d, v28.2d, #39 // .......................*................................................................ + // xar v13.2d, v19.2d, v27.2d, #56 // ........................*............................................................... + // xar v19.2d, v23.2d, v28.2d, #8 // ..........................*............................................................. + // xar v23.2d, v15.2d, v26.2d, #23 // ...........................*............................................................ + // xar v15.2d, v1.2d, v25.2d, #63 // ............................*........................................................... + // xar v1.2d, v8.2d, v28.2d, #9 // .............................*.......................................................... + // xar v8.2d, v16.2d, v25.2d, #19 // ...............................*........................................................ + // xar v16.2d, v7.2d, v29.2d, #58 // ................................*....................................................... + // xar v7.2d, v10.2d, v26.2d, #61 // .................................*...................................................... + // xar v10.2d, v3.2d, v28.2d, #36 // ..................................*..................................................... + // xar v3.2d, v18.2d, v28.2d, #43 // ....................................*................................................... + // xar v18.2d, v17.2d, v29.2d, #49 // .....................................*.................................................. + // xar v17.2d, v11.2d, v25.2d, #54 // ......................................*................................................. + // xar v11.2d, v9.2d, v27.2d, #44 // ........................................*............................................... + // xar v9.2d, v22.2d, v29.2d, #3 // .........................................*.............................................. + // xar v22.2d, v14.2d, v27.2d, #25 // ..........................................*............................................. + // xar v14.2d, v20.2d, v26.2d, #46 // ...........................................*............................................ + // xar v20.2d, v4.2d, v27.2d, #37 // .............................................*.......................................... + // xar v4.2d, v24.2d, v27.2d, #50 // ..............................................*......................................... + // xar v24.2d, v21.2d, v25.2d, #62 // ...............................................*........................................ + // xar v21.2d, v5.2d, v26.2d, #28 // ................................................*....................................... + // xar v27.2d, v6.2d, v25.2d, #20 // ..................................................*..................................... + // ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ....................................................................................*... + // ld1r {v28.2d}, [x30], #8 // ....................................................................................*... + // str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .....................................................................................*.. + // bcax v5.16b, v10.16b, v7.16b, v11.16b // ...................................................*.................................... + // bcax v6.16b, v11.16b, v8.16b, v7.16b // ....................................................*................................... + // bcax v7.16b, v7.16b, v9.16b, v8.16b // ......................................................*................................. + // bcax v8.16b, v8.16b, v10.16b, v9.16b // .......................................................*................................ + // bcax v9.16b, v9.16b, v11.16b, v10.16b // ........................................................*............................... + // bcax v10.16b, v15.16b, v12.16b, v16.16b // .........................................................*.............................. + // bcax v11.16b, v16.16b, v13.16b, v12.16b // ...........................................................*............................ + // bcax v12.16b, v12.16b, v14.16b, v13.16b // ............................................................*........................... + // bcax v13.16b, v13.16b, v15.16b, v14.16b // .............................................................*.......................... + // bcax v14.16b, v14.16b, v16.16b, v15.16b // ..............................................................*......................... + // bcax v15.16b, v20.16b, v17.16b, v21.16b // ................................................................*....................... + // bcax v16.16b, v21.16b, v18.16b, v17.16b // .................................................................*...................... + // bcax v17.16b, v17.16b, v19.16b, v18.16b // ..................................................................*..................... + // bcax v18.16b, v18.16b, v20.16b, v19.16b // ....................................................................*................... + // bcax v19.16b, v19.16b, v21.16b, v20.16b // .....................................................................*.................. + // bcax v20.16b, v0.16b, v22.16b, v1.16b // ......................................................................*................. + // bcax v21.16b, v1.16b, v23.16b, v22.16b // .......................................................................*................ + // bcax v22.16b, v22.16b, v24.16b, v23.16b // .........................................................................*.............. + // bcax v23.16b, v23.16b, v0.16b, v24.16b // ..........................................................................*............. + // bcax v24.16b, v24.16b, v1.16b, v0.16b // ...........................................................................*............ + // bcax v0.16b, v30.16b, v2.16b, v27.16b // .............................................................................*.......... + // bcax v1.16b, v27.16b, v3.16b, v2.16b // ..............................................................................*......... + // bcax v2.16b, v2.16b, v4.16b, v3.16b // ...............................................................................*........ + // bcax v3.16b, v3.16b, v30.16b, v4.16b // ................................................................................*....... + // bcax v4.16b, v4.16b, v27.16b, v30.16b // ..................................................................................*..... + // eor v0.16b, v0.16b, v28.16b // .......................................................................................* + + mld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + // Instructions: 181 + // Expected cycles: 90 + // Expected IPC: 2.01 + // + // ----------------------------------- cycle (expected) ------------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|-------------- + eor x0, x15, x11, ror #52 // *......................................................................................... // @slothy:interleaving_class=0 + eor x0, x0, x13, ror #48 // *......................................................................................... // @slothy:interleaving_class=0 + eor3 v30.16B, v0.16B, v5.16B, v10.16B // .*........................................................................................ // @slothy:interleaving_class=1 + eor3 v30.16B, v30.16B, v15.16B, v20.16B // .*........................................................................................ // @slothy:interleaving_class=1 + eor x26, x8, x9, ror #57 // ..*....................................................................................... // @slothy:interleaving_class=0 + eor x27, x0, x14, ror #10 // ..*....................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v1.16B, v6.16B, v11.16B // ...*...................................................................................... // @slothy:interleaving_class=1 + eor x29, x16, x28, ror #63 // ...*...................................................................................... // @slothy:interleaving_class=0 + eor3 v29.16B, v29.16B, v16.16B, v21.16B // ....*..................................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x6, ror #51 // ....*..................................................................................... // @slothy:interleaving_class=0 + eor x30, x23, x22, ror #50 // .....*.................................................................................... // @slothy:interleaving_class=0 + eor3 v28.16B, v2.16B, v7.16B, v12.16B // .....*.................................................................................... // @slothy:interleaving_class=1 + eor x0, x26, x10, ror #31 // ......*................................................................................... // @slothy:interleaving_class=0 + eor x29, x29, x19, ror #37 // ......*................................................................................... // @slothy:interleaving_class=0 + eor3 v28.16B, v28.16B, v17.16B, v22.16B // .......*.................................................................................. // @slothy:interleaving_class=1 + eor x27, x27, x12, ror #5 // .......*.................................................................................. // @slothy:interleaving_class=0 + eor3 v27.16B, v3.16B, v8.16B, v13.16B // ........*................................................................................. // @slothy:interleaving_class=1 + eor x30, x30, x24, ror #34 // ........*................................................................................. // @slothy:interleaving_class=0 + eor x0, x0, x7, ror #27 // .........*................................................................................ // @slothy:interleaving_class=0 + eor3 v27.16B, v27.16B, v18.16B, v23.16B // .........*................................................................................ // @slothy:interleaving_class=1 + eor x26, x30, x21, ror #26 // ..........*............................................................................... // @slothy:interleaving_class=0 + eor3 v26.16B, v4.16B, v9.16B, v14.16B // ..........*............................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x25, ror #15 // ...........*.............................................................................. // @slothy:interleaving_class=0 + ror x30, x27, #62 // ...........*.............................................................................. // @slothy:interleaving_class=0 + eor3 v26.16B, v26.16B, v19.16B, v24.16B // ............*............................................................................. // @slothy:interleaving_class=1 + eor x30, x30, x26, ror #57 // ............*............................................................................. // @slothy:interleaving_class=0 + ror x26, x26, #58 // .............*............................................................................ // @slothy:interleaving_class=0 + rax1 v25.2D, v30.2D, v28.2D // .............*............................................................................ // @slothy:interleaving_class=1 + eor x16, x30, x16 // ..............*........................................................................... // @slothy:interleaving_class=0 + rax1 v28.2D, v28.2D, v26.2D // ..............*........................................................................... // @slothy:interleaving_class=1 + eor x28, x30, x28, ror #63 // ...............*.......................................................................... // @slothy:interleaving_class=0 + str x28, [sp, #STACK_LOC_0] // ...............*.......................................................................... // @slothy:writes=stack_0 // @slothy:interleaving_class=0 + rax1 v26.2D, v26.2D, v29.2D // ................*......................................................................... // @slothy:interleaving_class=1 + eor x29, x29, x17, ror #36 // ................*......................................................................... // @slothy:interleaving_class=0 + eor x28, x1, x2, ror #61 // .................*........................................................................ // @slothy:interleaving_class=0 + rax1 v29.2D, v29.2D, v27.2D // .................*........................................................................ // @slothy:interleaving_class=1 + eor x19, x30, x19, ror #37 // ..................*....................................................................... // @slothy:interleaving_class=0 + rax1 v27.2D, v27.2D, v30.2D // ..................*....................................................................... // @slothy:interleaving_class=1 + eor x29, x29, x20, ror #2 // ...................*...................................................................... // @slothy:interleaving_class=0 + eor x28, x28, x4, ror #54 // ...................*...................................................................... // @slothy:interleaving_class=0 + eor v30.16B, v0.16B, v26.16B // ....................*..................................................................... // @slothy:interleaving_class=1 + eor x26, x26, x0, ror #55 // ....................*..................................................................... // @slothy:interleaving_class=0 + xar v0.2D, v2.2D, v29.2D, #2 // .....................*.................................................................... // @slothy:interleaving_class=1 + eor x28, x28, x3, ror #39 // .....................*.................................................................... // @slothy:interleaving_class=0 + eor x28, x28, x5, ror #25 // ......................*................................................................... // @slothy:interleaving_class=0 + xar v2.2D, v12.2D, v29.2D, #21 // ......................*................................................................... // @slothy:interleaving_class=1 + ror x0, x0, #56 // .......................*.................................................................. // @slothy:interleaving_class=0 + eor x0, x0, x29, ror #63 // .......................*.................................................................. // @slothy:interleaving_class=0 + xar v12.2D, v13.2D, v28.2D, #39 // ........................*................................................................. // @slothy:interleaving_class=1 + eor x27, x28, x27, ror #61 // ........................*................................................................. // @slothy:interleaving_class=0 + xar v13.2D, v19.2D, v27.2D, #56 // .........................*................................................................ // @slothy:interleaving_class=1 + eor x13, x0, x13, ror #46 // .........................*................................................................ // @slothy:interleaving_class=0 + eor x28, x29, x28, ror #63 // ..........................*............................................................... // @slothy:interleaving_class=0 + xar v19.2D, v23.2D, v28.2D, #8 // ..........................*............................................................... // @slothy:interleaving_class=1 + eor x29, x30, x20, ror #2 // ...........................*.............................................................. // @slothy:interleaving_class=0 + eor x20, x26, x3, ror #39 // ...........................*.............................................................. // @slothy:interleaving_class=0 + xar v23.2D, v15.2D, v26.2D, #23 // ............................*............................................................. // @slothy:interleaving_class=1 + eor x11, x0, x11, ror #50 // ............................*............................................................. // @slothy:interleaving_class=0 + xar v15.2D, v1.2D, v25.2D, #63 // .............................*............................................................ // @slothy:interleaving_class=1 + eor x25, x28, x25, ror #9 // .............................*............................................................ // @slothy:interleaving_class=0 + eor x3, x28, x21, ror #20 // ..............................*........................................................... // @slothy:interleaving_class=0 + xar v1.2D, v8.2D, v28.2D, #9 // ..............................*........................................................... // @slothy:interleaving_class=1 + eor x21, x26, x1 // ...............................*.......................................................... // @slothy:interleaving_class=0 + xar v8.2D, v16.2D, v25.2D, #19 // ...............................*.......................................................... // @slothy:interleaving_class=1 + eor x9, x27, x9, ror #49 // ................................*......................................................... // @slothy:interleaving_class=0 + eor x24, x28, x24, ror #28 // ................................*......................................................... // @slothy:interleaving_class=0 + xar v16.2D, v7.2D, v29.2D, #58 // .................................*........................................................ // @slothy:interleaving_class=1 + eor x1, x30, x17, ror #36 // .................................*........................................................ // @slothy:interleaving_class=0 + eor x14, x0, x14, ror #8 // ..................................*....................................................... // @slothy:interleaving_class=0 + xar v7.2D, v10.2D, v26.2D, #61 // ..................................*....................................................... // @slothy:interleaving_class=1 + eor x22, x28, x22, ror #44 // ...................................*...................................................... // @slothy:interleaving_class=0 + xar v10.2D, v3.2D, v28.2D, #36 // ...................................*...................................................... // @slothy:interleaving_class=1 + eor x8, x27, x8, ror #56 // ....................................*..................................................... // @slothy:interleaving_class=0 + eor x17, x27, x7, ror #19 // ....................................*..................................................... // @slothy:interleaving_class=0 + xar v3.2D, v18.2D, v28.2D, #43 // .....................................*.................................................... // @slothy:interleaving_class=1 + eor x15, x0, x15, ror #62 // .....................................*.................................................... // @slothy:interleaving_class=0 + bic x7, x20, x22, ror #47 // ......................................*................................................... // @slothy:interleaving_class=0 + xar v18.2D, v17.2D, v29.2D, #49 // ......................................*................................................... // @slothy:interleaving_class=1 + eor x4, x26, x4, ror #54 // .......................................*.................................................. // @slothy:interleaving_class=0 + xar v17.2D, v11.2D, v25.2D, #54 // .......................................*.................................................. // @slothy:interleaving_class=1 + eor x0, x0, x12, ror #3 // ........................................*................................................. // @slothy:interleaving_class=0 + eor x28, x28, x23, ror #58 // ........................................*................................................. // @slothy:interleaving_class=0 + xar v11.2D, v9.2D, v27.2D, #44 // .........................................*................................................ // @slothy:interleaving_class=1 + eor x23, x26, x2, ror #61 // .........................................*................................................ // @slothy:interleaving_class=0 + xar v9.2D, v22.2D, v29.2D, #3 // ..........................................*............................................... // @slothy:interleaving_class=1 + eor x26, x26, x5, ror #25 // ..........................................*............................................... // @slothy:interleaving_class=0 + eor x2, x7, x16, ror #39 // ...........................................*.............................................. // @slothy:interleaving_class=0 + xar v22.2D, v14.2D, v27.2D, #25 // ...........................................*.............................................. // @slothy:interleaving_class=1 + bic x7, x9, x20, ror #42 // ............................................*............................................. // @slothy:interleaving_class=0 + bic x30, x15, x9, ror #16 // ............................................*............................................. // @slothy:interleaving_class=0 + xar v14.2D, v20.2D, v26.2D, #46 // .............................................*............................................ // @slothy:interleaving_class=1 + eor x7, x7, x22, ror #25 // .............................................*............................................ // @slothy:interleaving_class=0 + xar v20.2D, v4.2D, v27.2D, #37 // ..............................................*........................................... // @slothy:interleaving_class=1 + eor x12, x30, x20, ror #58 // ..............................................*........................................... // @slothy:interleaving_class=0 + bic x20, x22, x16, ror #56 // ...............................................*.......................................... // @slothy:interleaving_class=0 + xar v4.2D, v24.2D, v27.2D, #50 // ...............................................*.......................................... // @slothy:interleaving_class=1 + eor x30, x27, x6, ror #43 // ................................................*......................................... // @slothy:interleaving_class=0 + eor x22, x20, x15, ror #23 // ................................................*......................................... // @slothy:interleaving_class=0 + xar v24.2D, v21.2D, v25.2D, #62 // .................................................*........................................ // @slothy:interleaving_class=1 + bic x6, x19, x13, ror #42 // .................................................*........................................ // @slothy:interleaving_class=0 + xar v21.2D, v5.2D, v26.2D, #28 // ..................................................*....................................... // @slothy:interleaving_class=1 + eor x6, x6, x17, ror #41 // ..................................................*....................................... // @slothy:interleaving_class=0 + bic x5, x13, x17, ror #63 // ...................................................*...................................... // @slothy:interleaving_class=0 + xar v27.2D, v6.2D, v25.2D, #20 // ...................................................*...................................... // @slothy:interleaving_class=1 + eor x5, x21, x5, ror #21 // ....................................................*..................................... // @slothy:interleaving_class=0 + bcax v5.16B, v10.16B, v7.16B, v11.16B // ....................................................*..................................... // @slothy:interleaving_class=1 + bic x17, x17, x21, ror #44 // .....................................................*.................................... // @slothy:interleaving_class=0 + eor x27, x27, x10, ror #23 // .....................................................*.................................... // @slothy:interleaving_class=0 + bcax v6.16B, v11.16B, v8.16B, v7.16B // ......................................................*................................... // @slothy:interleaving_class=1 + bic x21, x21, x25, ror #50 // ......................................................*................................... // @slothy:interleaving_class=0 + bic x20, x27, x4, ror #25 // .......................................................*.................................. // @slothy:interleaving_class=0 + bcax v7.16B, v7.16B, v9.16B, v8.16B // .......................................................*.................................. // @slothy:interleaving_class=1 + bic x10, x16, x15, ror #31 // ........................................................*................................. // @slothy:interleaving_class=0 + bcax v8.16B, v8.16B, v10.16B, v9.16B // ........................................................*................................. // @slothy:interleaving_class=1 + eor x16, x21, x19, ror #43 // .........................................................*................................ // @slothy:interleaving_class=0 + eor x21, x17, x25, ror #30 // .........................................................*................................ // @slothy:interleaving_class=0 + bcax v9.16B, v9.16B, v11.16B, v10.16B // ..........................................................*............................... // @slothy:interleaving_class=1 + bic x19, x25, x19, ror #57 // ..........................................................*............................... // @slothy:interleaving_class=0 + ldr x25, [sp, #STACK_OFFSET_COUNT] // ...........................................................*.............................. // @slothy:reads=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v10.16B, v15.16B, v12.16B, v16.16B // ...........................................................*.............................. // @slothy:interleaving_class=1 + eor x17, x10, x9, ror #47 // ............................................................*............................. // @slothy:interleaving_class=0 + bcax v11.16B, v16.16B, v13.16B, v12.16B // ............................................................*............................. // @slothy:interleaving_class=1 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // .............................................................*............................ // @slothy:interleaving_class=0 + eor x15, x20, x28, ror #27 // .............................................................*............................ // @slothy:interleaving_class=0 + bcax v12.16B, v12.16B, v14.16B, v13.16B // ..............................................................*........................... // @slothy:interleaving_class=1 + bic x20, x4, x28, ror #2 // ..............................................................*........................... // @slothy:interleaving_class=0 + bcax v13.16B, v13.16B, v15.16B, v14.16B // ...............................................................*.......................... // @slothy:interleaving_class=1 + eor x10, x20, x1, ror #50 // ...............................................................*.......................... // @slothy:interleaving_class=0 + bic x20, x11, x27, ror #60 // ................................................................*......................... // @slothy:interleaving_class=0 + bcax v14.16B, v14.16B, v16.16B, v15.16B // ................................................................*......................... // @slothy:interleaving_class=1 + eor x20, x20, x4, ror #21 // .................................................................*........................ // @slothy:interleaving_class=0 + bic x4, x28, x1, ror #48 // .................................................................*........................ // @slothy:interleaving_class=0 + bcax v15.16B, v20.16B, v17.16B, v21.16B // ..................................................................*....................... // @slothy:interleaving_class=1 + bic x1, x1, x11, ror #57 // ..................................................................*....................... // @slothy:interleaving_class=0 + bcax v16.16B, v21.16B, v18.16B, v17.16B // ...................................................................*...................... // @slothy:interleaving_class=1 + ldr x28, [x9, x25, lsl #3] // ...................................................................*...................... // @slothy:interleaving_class=0 + ldr x9, [sp, #STACK_LOC_0] // ....................................................................*..................... // @slothy:reads=stack_0 // @slothy:interleaving_class=0 + bcax v17.16B, v17.16B, v19.16B, v18.16B // ....................................................................*..................... // @slothy:interleaving_class=1 + add x25, x25, #1 // .....................................................................*.................... // @slothy:interleaving_class=0 + str x25, [sp, #STACK_OFFSET_COUNT] // .....................................................................*.................... // @slothy:writes=STACK_OFFSET_COUNT // @slothy:interleaving_class=0 + bcax v18.16B, v18.16B, v20.16B, v19.16B // ......................................................................*................... // @slothy:interleaving_class=1 + cmp x25, #(KECCAK_F1600_ROUNDS-1) // ......................................................................*................... // @slothy:ignore_useless_output // @slothy:interleaving_class=0 + bcax v19.16B, v19.16B, v21.16B, v20.16B // .......................................................................*.................. // @slothy:interleaving_class=1 + eor x25, x1, x27, ror #53 // .......................................................................*.................. // @slothy:interleaving_class=0 + bic x27, x30, x26, ror #47 // ........................................................................*................. // @slothy:interleaving_class=0 + bcax v20.16B, v0.16B, v22.16B, v1.16B // ........................................................................*................. // @slothy:interleaving_class=1 + eor x1, x5, x28 // .........................................................................*................ // @slothy:interleaving_class=0 + bcax v21.16B, v1.16B, v23.16B, v22.16B // .........................................................................*................ // @slothy:interleaving_class=1 + eor x5, x4, x11, ror #41 // ..........................................................................*............... // @slothy:interleaving_class=0 + eor x11, x19, x13, ror #35 // ..........................................................................*............... // @slothy:interleaving_class=0 + bcax v22.16B, v22.16B, v24.16B, v23.16B // ...........................................................................*.............. // @slothy:interleaving_class=1 + bic x13, x26, x24, ror #10 // ...........................................................................*.............. // @slothy:interleaving_class=0 + eor x28, x27, x24, ror #57 // ............................................................................*............. // @slothy:interleaving_class=0 + bcax v23.16B, v23.16B, v0.16B, v24.16B // ............................................................................*............. // @slothy:interleaving_class=1 + bic x27, x24, x9, ror #47 // .............................................................................*............ // @slothy:interleaving_class=0 + bcax v24.16B, v24.16B, v1.16B, v0.16B // .............................................................................*............ // @slothy:interleaving_class=1 + bic x19, x23, x3, ror #9 // ..............................................................................*........... // @slothy:interleaving_class=0 + bic x4, x29, x14, ror #41 // ..............................................................................*........... // @slothy:interleaving_class=0 + bcax v0.16B, v30.16B, v2.16B, v27.16B // ...............................................................................*.......... // @slothy:interleaving_class=1 + eor x24, x19, x29, ror #44 // ...............................................................................*.......... // @slothy:interleaving_class=0 + bic x29, x3, x29, ror #35 // ................................................................................*......... // @slothy:interleaving_class=0 + bcax v1.16B, v27.16B, v3.16B, v2.16B // ................................................................................*......... // @slothy:interleaving_class=1 + eor x13, x13, x9, ror #57 // .................................................................................*........ // @slothy:interleaving_class=0 + bcax v2.16B, v2.16B, v4.16B, v3.16B // .................................................................................*........ // @slothy:interleaving_class=1 + eor x19, x29, x14, ror #12 // ..................................................................................*....... // @slothy:interleaving_class=0 + bic x29, x9, x0, ror #19 // ..................................................................................*....... // @slothy:interleaving_class=0 + bcax v3.16B, v3.16B, v30.16B, v4.16B // ...................................................................................*...... // @slothy:interleaving_class=1 + bic x14, x14, x8, ror #5 // ...................................................................................*...... // @slothy:interleaving_class=0 + bcax v4.16B, v4.16B, v27.16B, v30.16B // ....................................................................................*..... // @slothy:interleaving_class=1 + eor x9, x14, x23, ror #43 // ....................................................................................*..... // @slothy:interleaving_class=0 + eor x14, x4, x8, ror #46 // .....................................................................................*.... // @slothy:interleaving_class=0 + bic x23, x8, x23, ror #38 // .....................................................................................*.... // @slothy:interleaving_class=0 + eor x8, x27, x0, ror #2 // ......................................................................................*... // @slothy:interleaving_class=0 + eor x4, x23, x3, ror #47 // ......................................................................................*... // @slothy:interleaving_class=0 + bic x3, x0, x30, ror #5 // .......................................................................................*.. // @slothy:interleaving_class=0 + eor x23, x3, x26, ror #52 // .......................................................................................*.. // @slothy:interleaving_class=0 + eor x3, x29, x30, ror #24 // ........................................................................................*. // @slothy:interleaving_class=0 + ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ........................................................................................*. // @slothy:reads=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + ld1r {v28.2D}, [x30], #8 // .........................................................................................* // @slothy:interleaving_class=1 + str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .........................................................................................* // @slothy:writes=STACK_OFFSET_CONST_VECTOR // @slothy:interleaving_class=1 + eor v0.16B, v0.16B, v28.16B // ..........................................................................................* // @slothy:interleaving_class=1 + + // ------------------------------------ cycle (expected) ------------------------------------> + // 0 25 50 75 + // |------------------------|------------------------|------------------------|--------------- + // eor x0, x15, x11, ror #52 // *.......................................................................................... + // eor x0, x0, x13, ror #48 // *.......................................................................................... + // eor x26, x8, x9, ror #57 // ..*........................................................................................ + // eor x27, x0, x14, ror #10 // ..*........................................................................................ + // eor x29, x16, x28, ror #63 // ...*....................................................................................... + // eor x26, x26, x6, ror #51 // ....*...................................................................................... + // eor x30, x23, x22, ror #50 // .....*..................................................................................... + // eor x0, x26, x10, ror #31 // ......*.................................................................................... + // eor x29, x29, x19, ror #37 // ......*.................................................................................... + // eor x27, x27, x12, ror #5 // .......*................................................................................... + // eor x30, x30, x24, ror #34 // ........*.................................................................................. + // eor x0, x0, x7, ror #27 // .........*................................................................................. + // eor x26, x30, x21, ror #26 // ..........*................................................................................ + // eor x26, x26, x25, ror #15 // ...........*............................................................................... + // ror x30, x27, #62 // ...........*............................................................................... + // eor x30, x30, x26, ror #57 // ............*.............................................................................. + // ror x26, x26, #58 // .............*............................................................................. + // eor x16, x30, x16 // ..............*............................................................................ + // eor x28, x30, x28, ror #63 // ...............*........................................................................... + // str x28, [sp, #STACK_LOC_0] // ...............*........................................................................... + // eor x29, x29, x17, ror #36 // ................*.......................................................................... + // eor x28, x1, x2, ror #61 // .................*......................................................................... + // eor x19, x30, x19, ror #37 // ..................*........................................................................ + // eor x29, x29, x20, ror #2 // ...................*....................................................................... + // eor x28, x28, x4, ror #54 // ...................*....................................................................... + // eor x26, x26, x0, ror #55 // ....................*...................................................................... + // eor x28, x28, x3, ror #39 // .....................*..................................................................... + // eor x28, x28, x5, ror #25 // ......................*.................................................................... + // ror x0, x0, #56 // .......................*................................................................... + // eor x0, x0, x29, ror #63 // .......................*................................................................... + // eor x27, x28, x27, ror #61 // ........................*.................................................................. + // eor x13, x0, x13, ror #46 // .........................*................................................................. + // eor x28, x29, x28, ror #63 // ..........................*................................................................ + // eor x29, x30, x20, ror #2 // ...........................*............................................................... + // eor x20, x26, x3, ror #39 // ...........................*............................................................... + // eor x11, x0, x11, ror #50 // ............................*.............................................................. + // eor x25, x28, x25, ror #9 // .............................*............................................................. + // eor x3, x28, x21, ror #20 // ..............................*............................................................ + // eor x21, x26, x1 // ...............................*........................................................... + // eor x9, x27, x9, ror #49 // ................................*.......................................................... + // eor x24, x28, x24, ror #28 // ................................*.......................................................... + // eor x1, x30, x17, ror #36 // .................................*......................................................... + // eor x14, x0, x14, ror #8 // ..................................*........................................................ + // eor x22, x28, x22, ror #44 // ...................................*....................................................... + // eor x8, x27, x8, ror #56 // ....................................*...................................................... + // eor x17, x27, x7, ror #19 // ....................................*...................................................... + // eor x15, x0, x15, ror #62 // .....................................*..................................................... + // bic x7, x20, x22, ror #47 // ......................................*.................................................... + // eor x4, x26, x4, ror #54 // .......................................*................................................... + // eor x0, x0, x12, ror #3 // ........................................*.................................................. + // eor x28, x28, x23, ror #58 // ........................................*.................................................. + // eor x23, x26, x2, ror #61 // .........................................*................................................. + // eor x26, x26, x5, ror #25 // ..........................................*................................................ + // eor x2, x7, x16, ror #39 // ...........................................*............................................... + // bic x7, x9, x20, ror #42 // ............................................*.............................................. + // bic x30, x15, x9, ror #16 // ............................................*.............................................. + // eor x7, x7, x22, ror #25 // .............................................*............................................. + // eor x12, x30, x20, ror #58 // ..............................................*............................................ + // bic x20, x22, x16, ror #56 // ...............................................*........................................... + // eor x30, x27, x6, ror #43 // ................................................*.......................................... + // eor x22, x20, x15, ror #23 // ................................................*.......................................... + // bic x6, x19, x13, ror #42 // .................................................*......................................... + // eor x6, x6, x17, ror #41 // ..................................................*........................................ + // bic x5, x13, x17, ror #63 // ...................................................*....................................... + // eor x5, x21, x5, ror #21 // ....................................................*...................................... + // bic x17, x17, x21, ror #44 // .....................................................*..................................... + // eor x27, x27, x10, ror #23 // .....................................................*..................................... + // bic x21, x21, x25, ror #50 // ......................................................*.................................... + // bic x20, x27, x4, ror #25 // .......................................................*................................... + // bic x10, x16, x15, ror #31 // ........................................................*.................................. + // eor x16, x21, x19, ror #43 // .........................................................*................................. + // eor x21, x17, x25, ror #30 // .........................................................*................................. + // bic x19, x25, x19, ror #57 // ..........................................................*................................ + // ldr x25, [sp, #STACK_OFFSET_COUNT] // ...........................................................*............................... + // eor x17, x10, x9, ror #47 // ............................................................*.............................. + // ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] // .............................................................*............................. + // eor x15, x20, x28, ror #27 // .............................................................*............................. + // bic x20, x4, x28, ror #2 // ..............................................................*............................ + // eor x10, x20, x1, ror #50 // ...............................................................*........................... + // bic x20, x11, x27, ror #60 // ................................................................*.......................... + // eor x20, x20, x4, ror #21 // .................................................................*......................... + // bic x4, x28, x1, ror #48 // .................................................................*......................... + // bic x1, x1, x11, ror #57 // ..................................................................*........................ + // ldr x28, [x9, x25, LSL #3] // ...................................................................*....................... + // ldr x9, [sp, #STACK_LOC_0] // ....................................................................*...................... + // add x25, x25, #1 // .....................................................................*..................... + // str x25, [sp, #STACK_OFFSET_COUNT] // .....................................................................*..................... + // cmp x25, #(KECCAK_F1600_ROUNDS-1) // ......................................................................*.................... + // eor x25, x1, x27, ror #53 // .......................................................................*................... + // bic x27, x30, x26, ror #47 // ........................................................................*.................. + // eor x1, x5, x28 // .........................................................................*................. + // eor x5, x4, x11, ror #41 // ..........................................................................*................ + // eor x11, x19, x13, ror #35 // ..........................................................................*................ + // bic x13, x26, x24, ror #10 // ...........................................................................*............... + // eor x28, x27, x24, ror #57 // ............................................................................*.............. + // bic x27, x24, x9, ror #47 // .............................................................................*............. + // bic x19, x23, x3, ror #9 // ..............................................................................*............ + // bic x4, x29, x14, ror #41 // ..............................................................................*............ + // eor x24, x19, x29, ror #44 // ...............................................................................*........... + // bic x29, x3, x29, ror #35 // ................................................................................*.......... + // eor x13, x13, x9, ror #57 // .................................................................................*......... + // eor x19, x29, x14, ror #12 // ..................................................................................*........ + // bic x29, x9, x0, ror #19 // ..................................................................................*........ + // bic x14, x14, x8, ror #5 // ...................................................................................*....... + // eor x9, x14, x23, ror #43 // ....................................................................................*...... + // eor x14, x4, x8, ror #46 // .....................................................................................*..... + // bic x23, x8, x23, ror #38 // .....................................................................................*..... + // eor x8, x27, x0, ror #2 // ......................................................................................*.... + // eor x4, x23, x3, ror #47 // ......................................................................................*.... + // bic x3, x0, x30, ror #5 // .......................................................................................*... + // eor x23, x3, x26, ror #52 // .......................................................................................*... + // eor x3, x29, x30, ror #24 // ........................................................................................*.. + // eor3 v30.16b, v0.16b, v5.16b, v10.16b // .*......................................................................................... + // eor3 v30.16b, v30.16b, v15.16b, v20.16b // .*......................................................................................... + // eor3 v29.16b, v1.16b, v6.16b, v11.16b // ...*....................................................................................... + // eor3 v29.16b, v29.16b, v16.16b, v21.16b // ....*...................................................................................... + // eor3 v28.16b, v2.16b, v7.16b, v12.16b // .....*..................................................................................... + // eor3 v28.16b, v28.16b, v17.16b, v22.16b // .......*................................................................................... + // eor3 v27.16b, v3.16b, v8.16b, v13.16b // ........*.................................................................................. + // eor3 v27.16b, v27.16b, v18.16b, v23.16b // .........*................................................................................. + // eor3 v26.16b, v4.16b, v9.16b, v14.16b // ..........*................................................................................ + // eor3 v26.16b, v26.16b, v19.16b, v24.16b // ............*.............................................................................. + // rax1 v25.2d, v30.2d, v28.2d // .............*............................................................................. + // rax1 v28.2d, v28.2d, v26.2d // ..............*............................................................................ + // rax1 v26.2d, v26.2d, v29.2d // ................*.......................................................................... + // rax1 v29.2d, v29.2d, v27.2d // .................*......................................................................... + // rax1 v27.2d, v27.2d, v30.2d // ..................*........................................................................ + // eor v30.16b, v0.16b, v26.16b // ....................*...................................................................... + // xar v0.2d, v2.2d, v29.2d, #2 // .....................*..................................................................... + // xar v2.2d, v12.2d, v29.2d, #21 // ......................*.................................................................... + // xar v12.2d, v13.2d, v28.2d, #39 // ........................*.................................................................. + // xar v13.2d, v19.2d, v27.2d, #56 // .........................*................................................................. + // xar v19.2d, v23.2d, v28.2d, #8 // ..........................*................................................................ + // xar v23.2d, v15.2d, v26.2d, #23 // ............................*.............................................................. + // xar v15.2d, v1.2d, v25.2d, #63 // .............................*............................................................. + // xar v1.2d, v8.2d, v28.2d, #9 // ..............................*............................................................ + // xar v8.2d, v16.2d, v25.2d, #19 // ...............................*........................................................... + // xar v16.2d, v7.2d, v29.2d, #58 // .................................*......................................................... + // xar v7.2d, v10.2d, v26.2d, #61 // ..................................*........................................................ + // xar v10.2d, v3.2d, v28.2d, #36 // ...................................*....................................................... + // xar v3.2d, v18.2d, v28.2d, #43 // .....................................*..................................................... + // xar v18.2d, v17.2d, v29.2d, #49 // ......................................*.................................................... + // xar v17.2d, v11.2d, v25.2d, #54 // .......................................*................................................... + // xar v11.2d, v9.2d, v27.2d, #44 // .........................................*................................................. + // xar v9.2d, v22.2d, v29.2d, #3 // ..........................................*................................................ + // xar v22.2d, v14.2d, v27.2d, #25 // ...........................................*............................................... + // xar v14.2d, v20.2d, v26.2d, #46 // .............................................*............................................. + // xar v20.2d, v4.2d, v27.2d, #37 // ..............................................*............................................ + // xar v4.2d, v24.2d, v27.2d, #50 // ...............................................*........................................... + // xar v24.2d, v21.2d, v25.2d, #62 // .................................................*......................................... + // xar v21.2d, v5.2d, v26.2d, #28 // ..................................................*........................................ + // xar v27.2d, v6.2d, v25.2d, #20 // ...................................................*....................................... + // ldr x30, [sp, #STACK_OFFSET_CONST_VECTOR] // ........................................................................................*.. + // ld1r {v28.2d}, [x30], #8 // .........................................................................................*. + // str x30, [sp, #STACK_OFFSET_CONST_VECTOR] // .........................................................................................*. + // bcax v5.16b, v10.16b, v7.16b, v11.16b // ....................................................*...................................... + // bcax v6.16b, v11.16b, v8.16b, v7.16b // ......................................................*.................................... + // bcax v7.16b, v7.16b, v9.16b, v8.16b // .......................................................*................................... + // bcax v8.16b, v8.16b, v10.16b, v9.16b // ........................................................*.................................. + // bcax v9.16b, v9.16b, v11.16b, v10.16b // ..........................................................*................................ + // bcax v10.16b, v15.16b, v12.16b, v16.16b // ...........................................................*............................... + // bcax v11.16b, v16.16b, v13.16b, v12.16b // ............................................................*.............................. + // bcax v12.16b, v12.16b, v14.16b, v13.16b // ..............................................................*............................ + // bcax v13.16b, v13.16b, v15.16b, v14.16b // ...............................................................*........................... + // bcax v14.16b, v14.16b, v16.16b, v15.16b // ................................................................*.......................... + // bcax v15.16b, v20.16b, v17.16b, v21.16b // ..................................................................*........................ + // bcax v16.16b, v21.16b, v18.16b, v17.16b // ...................................................................*....................... + // bcax v17.16b, v17.16b, v19.16b, v18.16b // ....................................................................*...................... + // bcax v18.16b, v18.16b, v20.16b, v19.16b // ......................................................................*.................... + // bcax v19.16b, v19.16b, v21.16b, v20.16b // .......................................................................*................... + // bcax v20.16b, v0.16b, v22.16b, v1.16b // ........................................................................*.................. + // bcax v21.16b, v1.16b, v23.16b, v22.16b // .........................................................................*................. + // bcax v22.16b, v22.16b, v24.16b, v23.16b // ...........................................................................*............... + // bcax v23.16b, v23.16b, v0.16b, v24.16b // ............................................................................*.............. + // bcax v24.16b, v24.16b, v1.16b, v0.16b // .............................................................................*............. + // bcax v0.16b, v30.16b, v2.16b, v27.16b // ...............................................................................*........... + // bcax v1.16b, v27.16b, v3.16b, v2.16b // ................................................................................*.......... + // bcax v2.16b, v2.16b, v4.16b, v3.16b // .................................................................................*......... + // bcax v3.16b, v3.16b, v30.16b, v4.16b // ...................................................................................*....... + // bcax v4.16b, v4.16b, v27.16b, v30.16b // ....................................................................................*...... + // eor v0.16b, v0.16b, v28.16b // ..........................................................................................* + + mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + + ble mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + final_scalar_rotate + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/* ***************** REGISTER DEALLOCATIONS ****************** */ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/aarch64/src/keccakf1600_round_constants.c b/dev/fips202/aarch64/src/keccakf1600_round_constants.c index eb578bcfcd..df0652c1f4 100644 --- a/dev/fips202/aarch64/src/keccakf1600_round_constants.c +++ b/dev/fips202/aarch64/src/keccakf1600_round_constants.c @@ -14,6 +14,7 @@ #if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \ defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \ defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) @@ -34,7 +35,8 @@ MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t #else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED */ @@ -42,6 +44,7 @@ MLD_EMPTY_CU(fips202_aarch64_round_constants) #endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/dev/fips202/aarch64/x3_v84a_scalar.h b/dev/fips202/aarch64/x3_v84a_scalar.h new file mode 100644 index 0000000000..fa7ad121eb --- /dev/null +++ b/dev/fips202/aarch64/x3_v84a_scalar.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H +#define MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X3 +/* Guard for assembly file */ +#define MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || + !mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm( + state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_DEV_FIPS202_AARCH64_X3_V84A_SCALAR_H */ diff --git a/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S b/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S new file mode 100644 index 0000000000..057d9e311b --- /dev/null +++ b/dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S @@ -0,0 +1,860 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. +// +// This is the SYMBOLIC (pre-SLOTHY) source. It must be optimized with SLOTHY +// (see the Makefile target keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S) +// to produce the interleaved production file. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_clean + Description: AArch64 hybrid scalar/vector clean implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations (clean version for optimization) + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t *state + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) +/* simpasm: header-end */ + +#define KECCAK_F1600_ROUNDS 24 + +/****************** REGISTER ALLOCATIONS *******************/ + + input_addr .req x0 + input_addr_hi .req x4 + input_rc .req x1 + const_addr .req x29 + cur_const .req x26 + + /* Mapping of Keccak-f1600 SIMD state to vector registers + * at the beginning and end of each round. Identical to x4: + * two independent states are packed into the 2 lanes of each + * 128-bit V register. */ + + vAba .req v0 + vAbe .req v1 + vAbi .req v2 + vAbo .req v3 + vAbu .req v4 + vAga .req v5 + vAge .req v6 + vAgi .req v7 + vAgo .req v8 + vAgu .req v9 + vAka .req v10 + vAke .req v11 + vAki .req v12 + vAko .req v13 + vAku .req v14 + vAma .req v15 + vAme .req v16 + vAmi .req v17 + vAmo .req v18 + vAmu .req v19 + vAsa .req v20 + vAse .req v21 + vAsi .req v22 + vAso .req v23 + vAsu .req v24 + + vAsud .req d24 + + tmp0 .req v25 + tmp1 .req v26 + tmp2 .req v27 + tmp3 .req v28 + + tmp0q .req q25 + tmp1q .req q26 + tmp2q .req q27 + tmp3q .req q28 + + tmp0d .req d25 + tmp2d .req d27 + + /* C[x] = A[x,0] xor A[x,1] xor A[x,2] xor A[x,3] xor A[x,4], for x in 0..4 */ + C0 .req v30 + C1 .req v29 + C2 .req v28 + C3 .req v27 + C4 .req v26 + + /* E[x] = C[x-1] xor rot(C[x+1],1), for x in 0..4 */ + E0 .req v26 + E1 .req v25 + E2 .req v29 + E3 .req v28 + E4 .req v27 + + /* A_[y,2*x+3*y] = rot(A[x,y]) */ + vAbi_ .req v2 + vAbo_ .req v3 + vAbu_ .req v4 + vAga_ .req v10 + vAge_ .req v11 + vAgi_ .req v7 + vAgo_ .req v8 + vAgu_ .req v9 + vAka_ .req v15 + vAke_ .req v16 + vAki_ .req v12 + vAko_ .req v13 + vAku_ .req v14 + vAma_ .req v20 + vAme_ .req v21 + vAmi_ .req v17 + vAmo_ .req v18 + vAmu_ .req v19 + vAsa_ .req v0 + vAse_ .req v1 + vAsi_ .req v22 + vAso_ .req v23 + vAsu_ .req v24 + vAba_ .req v30 + vAbe_ .req v27 + + /* Mapping of Keccak-f1600 state to scalar registers + * at the beginning and end of each round. */ + tAba .req x1 /* Some compilers don't like sAba */ + sAbe .req x6 + sAbi .req x11 + sAbo .req x16 + sAbu .req x21 + sAga .req x2 + sAge .req x7 + sAgi .req x12 + sAgo .req x17 + sAgu .req x22 + sAka .req x3 + sAke .req x8 + sAki .req x13 + sAko .req x28 + sAku .req x23 + sAma .req x4 + sAme .req x9 + sAmi .req x14 + sAmo .req x19 + sAmu .req x24 + sAsa .req x5 + sAse .req x10 + sAsi .req x15 + sAso .req x20 + sAsu .req x25 + + tmp .req x30 + +/************************ MACROS ****************************/ + +.macro eor3_m0 d, s0, s1, s2 + eor3 \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro rax1_m0 d, s0, s1 + rax1 \d\().2d, \s0\().2d, \s1\().2d +.endm + +.macro xar_m0 d, s0, s1, imm + xar \d\().2d, \s0\().2d, \s1\().2d, #\imm +.endm + +.macro bcax_m0 d, s0, s1, s2 + bcax \d\().16b, \s0\().16b, \s1\().16b, \s2\().16b +.endm + +.macro load_lane out0, out1, out2, out3 + ldp tmp0q, tmp1q, [input_addr], #32 + ld1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 + trn1 \out0\().2d, tmp0.2d, tmp2.2d + trn2 \out1\().2d, tmp0.2d, tmp2.2d + trn1 \out2\().2d, tmp1.2d, tmp3.2d + trn2 \out3\().2d, tmp1.2d, tmp3.2d +.endm + +.macro load_lane_single out + ldr tmp0d, [input_addr] + ldr tmp2d, [input_addr_hi] + trn1 \out\().2d, tmp0.2d, tmp2.2d +.endm + +.macro store_lane out0, out1, out2, out3 + trn1 tmp0\().2d, \out0\().2d, \out1\().2d + trn1 tmp1\().2d, \out2\().2d, \out3\().2d + stp tmp0q, tmp1q, [input_addr], #32 + trn2 tmp2\().2d, \out0\().2d, \out1\().2d + trn2 tmp3\().2d, \out2\().2d, \out3\().2d + st1 {tmp2.2d, tmp3.2d}, [input_addr_hi], #32 +.endm + +.macro store_lane_single out + str \out\()d, [input_addr] + trn2 tmp0.2d, \out\().2d, \out\().2d + str tmp0d, [input_addr_hi] +.endm + +/* Vector states are state0 (at input_addr) and state1 (at input_addr+0xc8). + * Identical layout to x4. */ +.macro load_input_vector + add input_addr_hi, input_addr, #0xc8 + load_lane vAba, vAbe, vAbi, vAbo + load_lane vAbu, vAga, vAge, vAgi + load_lane vAgo, vAgu, vAka, vAke + load_lane vAki, vAko, vAku, vAma + load_lane vAme, vAmi, vAmo, vAmu + load_lane vAsa, vAse, vAsi, vAso + load_lane_single vAsu + sub input_addr, input_addr, #0xc0 +.endm + +.macro store_input_vector + add input_addr_hi, input_addr, #0xc8 + store_lane vAba, vAbe, vAbi, vAbo + store_lane vAbu, vAga, vAge, vAgi + store_lane vAgo, vAgu, vAka, vAke + store_lane vAki, vAko, vAku, vAma + store_lane vAme, vAmi, vAmo, vAmu + store_lane vAsa, vAse, vAsi, vAso + store_lane_single vAsu +.endm + +/* The single scalar state is state2, at offset (2)*25*8 from input_addr. + * x3 only ever uses idx=0 (there is no second scalar pass). */ +.macro store_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + stp tAba, sAbe, [input_addr, #(1*8*0)] + stp sAbi, sAbo, [input_addr, #(1*8*2)] + stp sAbu, sAga, [input_addr, #(1*8*4)] + stp sAge, sAgi, [input_addr, #(1*8*6)] + stp sAgo, sAgu, [input_addr, #(1*8*8)] + stp sAka, sAke, [input_addr, #(1*8*10)] + stp sAki, sAko, [input_addr, #(1*8*12)] + stp sAku, sAma, [input_addr, #(1*8*14)] + stp sAme, sAmi, [input_addr, #(1*8*16)] + stp sAmo, sAmu, [input_addr, #(1*8*18)] + stp sAsa, sAse, [input_addr, #(1*8*20)] + stp sAsi, sAso, [input_addr, #(1*8*22)] + str sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +.macro load_input_scalar idx + add input_addr, input_addr, #((2 + \idx)*25*8) + ldp tAba, sAbe, [input_addr, #(1*8*0)] + ldp sAbi, sAbo, [input_addr, #(1*8*2)] + ldp sAbu, sAga, [input_addr, #(1*8*4)] + ldp sAge, sAgi, [input_addr, #(1*8*6)] + ldp sAgo, sAgu, [input_addr, #(1*8*8)] + ldp sAka, sAke, [input_addr, #(1*8*10)] + ldp sAki, sAko, [input_addr, #(1*8*12)] + ldp sAku, sAma, [input_addr, #(1*8*14)] + ldp sAme, sAmi, [input_addr, #(1*8*16)] + ldp sAmo, sAmu, [input_addr, #(1*8*18)] + ldp sAsa, sAse, [input_addr, #(1*8*20)] + ldp sAsi, sAso, [input_addr, #(1*8*22)] + ldr sAsu, [input_addr, #(1*8*24)] + sub input_addr, input_addr, #((2 + \idx)*25*8) +.endm + +#define STACK_LOCS 2 + +#define STACK_SIZE (16*6 + 8*8 + 6*8 + (STACK_LOCS) * 8) +#define STACK_BASE_GPRS (6*8) +#define STACK_BASE_VREGS (6*8 + 16*6) +#define STACK_OFFSET_LOCS (16*6 + 8*8 + 6*8) + +#define STACK_OFFSET_INPUT (0*8) +#define STACK_OFFSET_CONST_SCALAR (1*8) +#define STACK_OFFSET_CONST_VECTOR (2*8) +#define STACK_OFFSET_COUNT (3*8) + +#define STACK_LOC_0 ((STACK_OFFSET_LOCS) + 0*8) +#define STACK_LOC_1 ((STACK_OFFSET_LOCS) + 1*8) + +.macro save_gprs + stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro restore_gprs + ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] + ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] + ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] + ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] + ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] + ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] +.endm + +.macro save_vregs + stp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + stp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + stp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + stp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro restore_vregs + ldp d8, d9, [sp,#(STACK_BASE_VREGS+0*16)] + ldp d10, d11, [sp,#(STACK_BASE_VREGS+1*16)] + ldp d12, d13, [sp,#(STACK_BASE_VREGS+2*16)] + ldp d14, d15, [sp,#(STACK_BASE_VREGS+3*16)] +.endm + +.macro alloc_stack + sub sp, sp, #(STACK_SIZE) +.endm + +.macro free_stack + add sp, sp, #(STACK_SIZE) +.endm + +.macro eor5 dst, src0, src1, src2, src3, src4 + eor \dst, \src0, \src1 + eor \dst, \dst, \src2 + eor \dst, \dst, \src3 + eor \dst, \dst, \src4 +.endm + +.macro chi_step_ror out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), X, \a\(), ror #\r2 +.endm + +.macro chi_step_ror2 out, a, b, c, r1, r2 + bic X, \b\(), \c\(), ror #\r1 + eor \out\(), \a\(), X, ror #\r2 +.endm + +// Scalar round macros are IDENTICAL to the x4 kernel; they operate on the +// single scalar state held in GPRs. +.macro scalar_round_initial + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #1 + bic x9, x0, x16, ror #9 + str x24, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 +.endm + +// Neon round: uses the single-instruction Armv8.4-A SHA3 forms (eor3, rax1, +// xar, bcax; the "_m0" macros) throughout. Requires __ARM_FEATURE_SHA3 +// (guarded above). +.macro vector_round + eor3_m0 C0, vAba, vAga, vAka + eor3_m0 C0, C0, vAma, vAsa + eor3_m0 C1, vAbe, vAge, vAke + eor3_m0 C1, C1, vAme, vAse + eor3_m0 C2, vAbi, vAgi, vAki + eor3_m0 C2, C2, vAmi, vAsi + eor3_m0 C3, vAbo, vAgo, vAko + eor3_m0 C3, C3, vAmo, vAso + eor3_m0 C4, vAbu, vAgu, vAku + eor3_m0 C4, C4, vAmu, vAsu + rax1_m0 E1, C0, C2 + rax1_m0 E3, C2, C4 + rax1_m0 E0, C4, C1 + rax1_m0 E2, C1, C3 + rax1_m0 E4, C3, C0 + eor vAba_.16b, vAba.16b, E0.16b + xar_m0 vAsa_, vAbi, E2, 2 + xar_m0 vAbi_, vAki, E2, 21 + xar_m0 vAki_, vAko, E3, 39 + xar_m0 vAko_, vAmu, E4, 56 + xar_m0 vAmu_, vAso, E3, 8 + xar_m0 vAso_, vAma, E0, 23 + xar_m0 vAka_, vAbe, E1, 63 + xar_m0 vAse_, vAgo, E3, 9 + xar_m0 vAgo_, vAme, E1, 19 + xar_m0 vAke_, vAgi, E2, 58 + xar_m0 vAgi_, vAka, E0, 61 + xar_m0 vAga_, vAbo, E3, 36 + xar_m0 vAbo_, vAmo, E3, 43 + xar_m0 vAmo_, vAmi, E2, 49 + xar_m0 vAmi_, vAke, E1, 54 + xar_m0 vAge_, vAgu, E4, 44 + xar_m0 vAgu_, vAsi, E2, 3 + xar_m0 vAsi_, vAku, E4, 25 + xar_m0 vAku_, vAsa, E0, 46 + xar_m0 vAma_, vAbu, E4, 37 + xar_m0 vAbu_, vAsu, E4, 50 + xar_m0 vAsu_, vAse, E1, 62 + xar_m0 vAme_, vAga, E0, 28 + xar_m0 vAbe_, vAge, E1, 20 + ldr tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:reads=STACK_OFFSET_CONST_VECTOR + ld1r {v28.2d}, [tmp], #8 + str tmp, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + bcax_m0 vAga, vAga_, vAgi_, vAge_ + bcax_m0 vAge, vAge_, vAgo_, vAgi_ + bcax_m0 vAgi, vAgi_, vAgu_, vAgo_ + bcax_m0 vAgo, vAgo_, vAga_, vAgu_ + bcax_m0 vAgu, vAgu_, vAge_, vAga_ + bcax_m0 vAka, vAka_, vAki_, vAke_ + bcax_m0 vAke, vAke_, vAko_, vAki_ + bcax_m0 vAki, vAki_, vAku_, vAko_ + bcax_m0 vAko, vAko_, vAka_, vAku_ + bcax_m0 vAku, vAku_, vAke_, vAka_ + bcax_m0 vAma, vAma_, vAmi_, vAme_ + bcax_m0 vAme, vAme_, vAmo_, vAmi_ + bcax_m0 vAmi, vAmi_, vAmu_, vAmo_ + bcax_m0 vAmo, vAmo_, vAma_, vAmu_ + bcax_m0 vAmu, vAmu_, vAme_, vAma_ + bcax_m0 vAsa, vAsa_, vAsi_, vAse_ + bcax_m0 vAse, vAse_, vAso_, vAsi_ + bcax_m0 vAsi, vAsi_, vAsu_, vAso_ + bcax_m0 vAso, vAso_, vAsa_, vAsu_ + bcax_m0 vAsu, vAsu_, vAse_, vAsa_ + bcax_m0 vAba, vAba_, vAbi_, vAbe_ + bcax_m0 vAbe, vAbe_, vAbo_, vAbi_ + bcax_m0 vAbi, vAbi_, vAbu_, vAbo_ + bcax_m0 vAbo, vAbo_, vAba_, vAbu_ + bcax_m0 vAbu, vAbu_, vAbe_, vAba_ + eor vAba.16b, vAba.16b, v28.16b +.endm + +.macro scalar_round_noninitial + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #62 + eor x30, x30, x26, ror #57 + ror x26, x26, #58 + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #STACK_LOC_0] // @slothy:writes=stack_0 + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #56 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #STACK_OFFSET_COUNT] // @slothy:reads=STACK_OFFSET_COUNT + eor x17, x10, x9, ror #47 + ldr x9, [sp, #STACK_OFFSET_CONST_SCALAR] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, LSL #3] + ldr x9, [sp, #STACK_LOC_0] // @slothy:reads=stack_0 + add x25, x25, #1 + str x25, [sp, #STACK_OFFSET_COUNT] // @slothy:writes=STACK_OFFSET_COUNT + cmp x25, #(KECCAK_F1600_ROUNDS-1) // @slothy:ignore_useless_output + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 +.endm + +.macro final_scalar_rotate + ror sAga, sAga,#(64-3) + ror sAka, sAka,#(64-25) + ror sAma, sAma,#(64-10) + ror sAsa, sAsa,#(64-39) + ror sAbe, sAbe,#(64-21) + ror sAge, sAge,#(64-45) + ror sAke, sAke,#(64-8) + ror sAme, sAme,#(64-15) + ror sAse, sAse,#(64-41) + ror sAbi, sAbi,#(64-14) + ror sAgi, sAgi,#(64-61) + ror sAki, sAki,#(64-18) + ror sAmi, sAmi,#(64-56) + ror sAsi, sAsi,#(64-2) + ror sAgo, sAgo,#(64-28) + ror sAko, sAko,#(64-1) + ror sAmo, sAmo,#(64-27) + ror sAso, sAso,#(64-62) + ror sAbu, sAbu,#(64-44) + ror sAgu, sAgu,#(64-20) + ror sAku, sAku,#(64-6) + ror sAmu, sAmu,#(64-36) + ror sAsu, sAsu,#(64-55) +.endm + + .text + .global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + .balign 4 +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + alloc_stack + save_gprs + save_vregs + + mov const_addr, input_rc + + str const_addr, [sp, #STACK_OFFSET_CONST_SCALAR] // @slothy:writes=STACK_OFFSET_CONST_SCALAR + str const_addr, [sp, #STACK_OFFSET_CONST_VECTOR] // @slothy:writes=STACK_OFFSET_CONST_VECTOR + str input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:writes=STACK_OFFSET_INPUT + + load_input_vector // Vector input (state0, state1) + load_input_scalar 0 // Scalar input (state2) + + // Single pass, 1:1 interleave. + // Initial iteration: scalar_round_initial does round 0 of the scalar state + // and sets the round counter to 1; vector_round does round 0 of the two + // vector states. Then the loop runs rounds 1..23 (23 iterations), one + // scalar + one vector round each, for a total of 24 rounds on all three + // states. +mld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + scalar_round_initial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + scalar_round_noninitial // @slothy:interleaving_class=0 + vector_round // @slothy:interleaving_class=1 +mld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + ble mld_keccak_f1600_x3_v84a_scalar_hybrid_loop + final_scalar_rotate + + ldr input_addr, [sp, #STACK_OFFSET_INPUT] // @slothy:reads=STACK_OFFSET_INPUT + store_input_scalar 0 + store_input_vector + + restore_vregs + restore_gprs + free_stack + ret + +/****************** REGISTER DEALLOCATIONS *******************/ + .unreq input_addr + .unreq input_addr_hi + .unreq input_rc + .unreq const_addr + .unreq cur_const + .unreq vAba + .unreq vAbe + .unreq vAbi + .unreq vAbo + .unreq vAbu + .unreq vAga + .unreq vAge + .unreq vAgi + .unreq vAgo + .unreq vAgu + .unreq vAka + .unreq vAke + .unreq vAki + .unreq vAko + .unreq vAku + .unreq vAma + .unreq vAme + .unreq vAmi + .unreq vAmo + .unreq vAmu + .unreq vAsa + .unreq vAse + .unreq vAsi + .unreq vAso + .unreq vAsu + .unreq vAsud + .unreq tmp0 + .unreq tmp1 + .unreq tmp2 + .unreq tmp3 + .unreq tmp0q + .unreq tmp1q + .unreq tmp2q + .unreq tmp3q + .unreq tmp0d + .unreq tmp2d + .unreq C0 + .unreq C1 + .unreq C2 + .unreq C3 + .unreq C4 + .unreq E0 + .unreq E1 + .unreq E2 + .unreq E3 + .unreq E4 + .unreq vAbi_ + .unreq vAbo_ + .unreq vAbu_ + .unreq vAga_ + .unreq vAge_ + .unreq vAgi_ + .unreq vAgo_ + .unreq vAgu_ + .unreq vAka_ + .unreq vAke_ + .unreq vAki_ + .unreq vAko_ + .unreq vAku_ + .unreq vAma_ + .unreq vAme_ + .unreq vAmi_ + .unreq vAmo_ + .unreq vAmu_ + .unreq vAsa_ + .unreq vAse_ + .unreq vAsi_ + .unreq vAso_ + .unreq vAsu_ + .unreq vAba_ + .unreq vAbe_ + .unreq tAba + .unreq sAbe + .unreq sAbi + .unreq sAbo + .unreq sAbu + .unreq sAga + .unreq sAge + .unreq sAgi + .unreq sAgo + .unreq sAgu + .unreq sAka + .unreq sAke + .unreq sAki + .unreq sAko + .unreq sAku + .unreq sAma + .unreq sAme + .unreq sAmi + .unreq sAmo + .unreq sAmu + .unreq sAsa + .unreq sAse + .unreq sAsi + .unreq sAso + .unreq sAsu + .unreq tmp + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef KECCAK_F1600_ROUNDS +#undef STACK_LOCS +#undef STACK_SIZE +#undef STACK_BASE_GPRS +#undef STACK_BASE_VREGS +#undef STACK_OFFSET_LOCS +#undef STACK_OFFSET_INPUT +#undef STACK_OFFSET_CONST_SCALAR +#undef STACK_OFFSET_CONST_VECTOR +#undef STACK_OFFSET_COUNT +#undef STACK_LOC_0 +#undef STACK_LOC_1 + +/* simpasm: footer-start */ +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/examples/basic_deterministic/mldsa_native/mldsa_native_config.h b/examples/basic_deterministic/mldsa_native/mldsa_native_config.h index 99f4062e34..ebf84c8a8f 100644 --- a/examples/basic_deterministic/mldsa_native/mldsa_native_config.h +++ b/examples/basic_deterministic/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/basic_lowram/mldsa_native/mldsa_native_config.h b/examples/basic_lowram/mldsa_native/mldsa_native_config.h index fa41d68f15..396fa3d538 100644 --- a/examples/basic_lowram/mldsa_native/mldsa_native_config.h +++ b/examples/basic_lowram/mldsa_native/mldsa_native_config.h @@ -413,6 +413,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h b/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h index 5da4634b8e..cec3342d88 100644 --- a/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h +++ b/examples/bring_your_own_fips202/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h b/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h index efb55b8c2f..9da43a32cc 100644 --- a/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h +++ b/examples/bring_your_own_fips202_static/mldsa_native/mldsa_native_config.h @@ -412,6 +412,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/context_parameter/mldsa_native/mldsa_native_config.h b/examples/context_parameter/mldsa_native/mldsa_native_config.h index c5e715109e..034a0e657d 100644 --- a/examples/context_parameter/mldsa_native/mldsa_native_config.h +++ b/examples/context_parameter/mldsa_native/mldsa_native_config.h @@ -417,6 +417,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/custom_backend/mldsa_native/mldsa_native_config.h b/examples/custom_backend/mldsa_native/mldsa_native_config.h index e8527151b2..3e628be55b 100644 --- a/examples/custom_backend/mldsa_native/mldsa_native_config.h +++ b/examples/custom_backend/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build/mldsa_native/mldsa_native_config.h b/examples/monolithic_build/mldsa_native/mldsa_native_config.h index a500982e38..5f8b2031d7 100644 --- a/examples/monolithic_build/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h index 7d4094b8df..b43576b386 100644 --- a/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_multilevel/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h index 3ee4024d6c..34eb274b5f 100644 --- a/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_multilevel_native/mldsa_native/mldsa_native_config.h @@ -411,6 +411,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h b/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h index a98f9e1f3d..44858e959c 100644 --- a/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h +++ b/examples/monolithic_build_native/mldsa_native/mldsa_native_config.h @@ -408,6 +408,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/multilevel_build/mldsa_native/mldsa_native_config.h b/examples/multilevel_build/mldsa_native/mldsa_native_config.h index 3231525631..09609ab84e 100644 --- a/examples/multilevel_build/mldsa_native/mldsa_native_config.h +++ b/examples/multilevel_build/mldsa_native/mldsa_native_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h b/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h index e1a04fb214..7e98ace116 100644 --- a/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h +++ b/examples/multilevel_build_native/mldsa_native/mldsa_native_config.h @@ -408,6 +408,29 @@ */ #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/examples/restartable_sign/mldsa_native/mldsa_native_config.h b/examples/restartable_sign/mldsa_native/mldsa_native_config.h index 39a7b40733..0d1e912851 100644 --- a/examples/restartable_sign/mldsa_native/mldsa_native_config.h +++ b/examples/restartable_sign/mldsa_native/mldsa_native_config.h @@ -412,6 +412,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/mldsa/mldsa_native.c b/mldsa/mldsa_native.c index 59a91ac2bc..00256a96ac 100644 --- a/mldsa/mldsa_native.c +++ b/mldsa/mldsa_native.c @@ -70,9 +70,10 @@ #if !defined(MLD_CONFIG_FIPS202_CUSTOM_HEADER) #include "src/fips202/fips202.c" +#include "src/fips202/fips202x3.c" #include "src/fips202/fips202x4.c" #include "src/fips202/keccakf1600.c" -#endif +#endif /* !MLD_CONFIG_FIPS202_CUSTOM_HEADER */ #if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) #if defined(MLD_SYS_AARCH64) @@ -510,6 +511,11 @@ #undef mld_xof128_init #undef mld_xof128_release #undef mld_xof128_squeezeblocks +#undef mld_xof128_x3_absorb +#undef mld_xof128_x3_ctx +#undef mld_xof128_x3_init +#undef mld_xof128_x3_release +#undef mld_xof128_x3_squeezeblocks #undef mld_xof128_x4_absorb #undef mld_xof128_x4_ctx #undef mld_xof128_x4_init @@ -520,6 +526,11 @@ #undef mld_xof256_init #undef mld_xof256_release #undef mld_xof256_squeezeblocks +#undef mld_xof256_x3_absorb +#undef mld_xof256_x3_ctx +#undef mld_xof256_x3_init +#undef mld_xof256_x3_release +#undef mld_xof256_x3_squeezeblocks #undef mld_xof256_x4_absorb #undef mld_xof256_x4_ctx #undef mld_xof256_x4_init @@ -554,6 +565,16 @@ #undef mld_shake256_init #undef mld_shake256_release #undef mld_shake256_squeeze +/* mldsa/src/fips202/fips202x3.h */ +#undef MLD_FIPS202_FIPS202X3_H +#undef mld_shake128x3_absorb_once +#undef mld_shake128x3_init +#undef mld_shake128x3_release +#undef mld_shake128x3_squeezeblocks +#undef mld_shake256x3_absorb_once +#undef mld_shake256x3_init +#undef mld_shake256x3_release +#undef mld_shake256x3_squeezeblocks /* mldsa/src/fips202/fips202x4.h */ #undef MLD_FIPS202_FIPS202X4_H #undef mld_shake128x4_absorb_once @@ -568,9 +589,13 @@ #undef MLD_FIPS202_KECCAKF1600_H #undef MLD_KECCAK_LANES #undef MLD_KECCAK_WAY +#undef MLD_KECCAK_WAY_X3 #undef mld_keccakf1600_extract_bytes #undef mld_keccakf1600_permute #undef mld_keccakf1600_xor_bytes +#undef mld_keccakf1600x3_extract_bytes +#undef mld_keccakf1600x3_permute +#undef mld_keccakf1600x3_xor_bytes #undef mld_keccakf1600x4_extract_bytes #undef mld_keccakf1600x4_permute #undef mld_keccakf1600x4_xor_bytes @@ -594,6 +619,7 @@ #undef mld_keccak_f1600_x1_scalar_aarch64_asm #undef mld_keccak_f1600_x1_v84a_aarch64_asm #undef mld_keccak_f1600_x2_v84a_aarch64_asm +#undef mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm #undef mld_keccakf1600_round_constants @@ -609,6 +635,10 @@ #undef MLD_FIPS202_AARCH64_NEED_X2_V84A #undef MLD_FIPS202_NATIVE_AARCH64_X2_V84A_H #undef MLD_USE_NATIVE_FIPS202_X4 +/* mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h */ +#undef MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID +#undef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#undef MLD_USE_NATIVE_FIPS202_X3 /* mldsa/src/fips202/native/aarch64/x4_v8a_scalar.h */ #undef MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID #undef MLD_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H diff --git a/mldsa/mldsa_native_asm.S b/mldsa/mldsa_native_asm.S index 4d6ad65878..b1cfcba85d 100644 --- a/mldsa/mldsa_native_asm.S +++ b/mldsa/mldsa_native_asm.S @@ -112,6 +112,7 @@ #include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S" #include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S" #endif /* MLD_SYS_AARCH64 */ @@ -548,6 +549,11 @@ #undef mld_xof128_init #undef mld_xof128_release #undef mld_xof128_squeezeblocks +#undef mld_xof128_x3_absorb +#undef mld_xof128_x3_ctx +#undef mld_xof128_x3_init +#undef mld_xof128_x3_release +#undef mld_xof128_x3_squeezeblocks #undef mld_xof128_x4_absorb #undef mld_xof128_x4_ctx #undef mld_xof128_x4_init @@ -558,6 +564,11 @@ #undef mld_xof256_init #undef mld_xof256_release #undef mld_xof256_squeezeblocks +#undef mld_xof256_x3_absorb +#undef mld_xof256_x3_ctx +#undef mld_xof256_x3_init +#undef mld_xof256_x3_release +#undef mld_xof256_x3_squeezeblocks #undef mld_xof256_x4_absorb #undef mld_xof256_x4_ctx #undef mld_xof256_x4_init @@ -592,6 +603,16 @@ #undef mld_shake256_init #undef mld_shake256_release #undef mld_shake256_squeeze +/* mldsa/src/fips202/fips202x3.h */ +#undef MLD_FIPS202_FIPS202X3_H +#undef mld_shake128x3_absorb_once +#undef mld_shake128x3_init +#undef mld_shake128x3_release +#undef mld_shake128x3_squeezeblocks +#undef mld_shake256x3_absorb_once +#undef mld_shake256x3_init +#undef mld_shake256x3_release +#undef mld_shake256x3_squeezeblocks /* mldsa/src/fips202/fips202x4.h */ #undef MLD_FIPS202_FIPS202X4_H #undef mld_shake128x4_absorb_once @@ -606,9 +627,13 @@ #undef MLD_FIPS202_KECCAKF1600_H #undef MLD_KECCAK_LANES #undef MLD_KECCAK_WAY +#undef MLD_KECCAK_WAY_X3 #undef mld_keccakf1600_extract_bytes #undef mld_keccakf1600_permute #undef mld_keccakf1600_xor_bytes +#undef mld_keccakf1600x3_extract_bytes +#undef mld_keccakf1600x3_permute +#undef mld_keccakf1600x3_xor_bytes #undef mld_keccakf1600x4_extract_bytes #undef mld_keccakf1600x4_permute #undef mld_keccakf1600x4_xor_bytes @@ -632,6 +657,7 @@ #undef mld_keccak_f1600_x1_scalar_aarch64_asm #undef mld_keccak_f1600_x1_v84a_aarch64_asm #undef mld_keccak_f1600_x2_v84a_aarch64_asm +#undef mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm #undef mld_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm #undef mld_keccakf1600_round_constants @@ -647,6 +673,10 @@ #undef MLD_FIPS202_AARCH64_NEED_X2_V84A #undef MLD_FIPS202_NATIVE_AARCH64_X2_V84A_H #undef MLD_USE_NATIVE_FIPS202_X4 +/* mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h */ +#undef MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID +#undef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#undef MLD_USE_NATIVE_FIPS202_X3 /* mldsa/src/fips202/native/aarch64/x4_v8a_scalar.h */ #undef MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID #undef MLD_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H diff --git a/mldsa/mldsa_native_config.h b/mldsa/mldsa_native_config.h index 1baba77f66..3e5c0636e7 100644 --- a/mldsa/mldsa_native_config.h +++ b/mldsa/mldsa_native_config.h @@ -395,6 +395,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/mldsa/src/fips202/fips202x3.c b/mldsa/src/fips202/fips202x3.c new file mode 100644 index 0000000000..181d889bce --- /dev/null +++ b/mldsa/src/fips202/fips202x3.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + +/* + * 3-way batched SHAKE128/SHAKE256 (MLD_CONFIG_USE_X3), mirroring fips202x4.c + * but 3-wide. See fips202x3.h. + */ + +#include "../common.h" +#if !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && defined(MLD_CONFIG_USE_X3) + +#include "../ct.h" +#include "fips202.h" +#include "fips202x3.h" +#include "keccakf1600.h" + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ + defined(MLD_UNIT_TEST) +static void mld_keccak_absorb_once_x3(uint64_t *s, unsigned r, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, size_t inlen, + uint8_t p) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLD_KECCAK_LANES) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3))) +{ + while (inlen >= r) + __loop__( + assigns(inlen, in0, in1, in2, memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + invariant(inlen <= loop_entry(inlen)) + invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen)) + invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen)) + invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen)) + decreases(inlen)) + { + mld_keccakf1600x3_xor_bytes(s, in0, in1, in2, 0, r); + mld_keccakf1600x3_permute(s); + + in0 += r; + in1 += r; + in2 += r; + inlen -= r; + } + + /* Safety: At this point, inlen < r, so the truncations to unsigned are safe + * below. */ + if (inlen > 0) + { + mld_keccakf1600x3_xor_bytes(s, in0, in1, in2, 0, (unsigned)inlen); + } + + if (inlen == r - 1) + { + p |= 128; + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, (unsigned)inlen, 1); + } + else + { + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, (unsigned)inlen, 1); + p = 128; + mld_keccakf1600x3_xor_bytes(s, &p, &p, &p, r - 1, 1); + } +} + +static void mld_keccak_squeezeblocks_x3(uint8_t *out0, uint8_t *out1, + uint8_t *out2, size_t nblocks, + uint64_t *s, unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLD_KECCAK_LANES) + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(out0, nblocks * r)) + requires(memory_no_alias(out1, nblocks * r)) + requires(memory_no_alias(out2, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + assigns(memory_slice(out0, nblocks * r)) + assigns(memory_slice(out1, nblocks * r)) + assigns(memory_slice(out2, nblocks * r))) +{ + size_t current_offset = 0; + while (nblocks > 0) + __loop__( + assigns(nblocks, current_offset, + memory_slice(s, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3), + memory_slice(out0, nblocks * r), + memory_slice(out1, nblocks * r), + memory_slice(out2, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks)) + invariant(current_offset == (loop_entry(nblocks) - nblocks) * r) + decreases(nblocks)) + { + mld_keccakf1600x3_permute(s); + mld_keccakf1600x3_extract_bytes(s, &out0[current_offset], + &out1[current_offset], + &out2[current_offset], 0, r); + current_offset += r; + nblocks--; + } +} +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ + +#if !defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +MLD_INTERNAL_API +void mld_shake128x3_absorb_once(mld_shake128x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +{ + mld_memset(state, 0, sizeof(mld_shake128x3ctx)); + mld_keccak_absorb_once_x3(state->ctx, SHAKE128_RATE, in0, in1, in2, inlen, + 0x1F); +} + +MLD_INTERNAL_API +void mld_shake128x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake128x3ctx *state) +{ + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, state->ctx, + SHAKE128_RATE); +} + +MLD_INTERNAL_API +void mld_shake128x3_init(mld_shake128x3ctx *state) { (void)state; } +MLD_INTERNAL_API +void mld_shake128x3_release(mld_shake128x3ctx *state) +{ + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(state, sizeof(mld_shake128x3ctx)); +} +#endif /* !MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || \ + (!defined(MLD_CONFIG_NO_SIGN_API) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST))) +MLD_INTERNAL_API +void mld_shake256x3_absorb_once(mld_shake256x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +{ + mld_memset(state, 0, sizeof(mld_shake256x3ctx)); + mld_keccak_absorb_once_x3(state->ctx, SHAKE256_RATE, in0, in1, in2, inlen, + 0x1F); +} + +MLD_INTERNAL_API +void mld_shake256x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake256x3ctx *state) +{ + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, state->ctx, + SHAKE256_RATE); +} + +MLD_INTERNAL_API +void mld_shake256x3_init(mld_shake256x3ctx *state) { (void)state; } +MLD_INTERNAL_API +void mld_shake256x3_release(mld_shake256x3ctx *state) +{ + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(state, sizeof(mld_shake256x3ctx)); +} +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ + +#endif /* !MLD_CONFIG_MULTILEVEL_NO_SHARED && !MLD_CONFIG_SERIAL_FIPS202_ONLY \ + && MLD_CONFIG_USE_X3 */ diff --git a/mldsa/src/fips202/fips202x3.h b/mldsa/src/fips202/fips202x3.h new file mode 100644 index 0000000000..523b85ad27 --- /dev/null +++ b/mldsa/src/fips202/fips202x3.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_FIPS202_FIPS202X3_H +#define MLD_FIPS202_FIPS202X3_H + +#include "../common.h" + +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && defined(MLD_CONFIG_USE_X3) + +#include + +#include "../cbmc.h" +#include "fips202.h" +#include "keccakf1600.h" + +/* + * 3-way batched SHAKE (MLD_CONFIG_USE_X3), mirroring fips202x4 but 3-wide. + * Provides SHAKE128 (x3 ExpandA: mld_poly_uniform_3x) and SHAKE256 (x3 + * ExpandS and ExpandMask: mld_poly_uniform_eta_3x, mld_poly_uniform_gamma1_3x). + * Mirrors the SHAKE128/SHAKE256 blocks in fips202x4.h. + */ + +/** Context for the non-incremental 3-way SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; /**< 3-way Keccak state, + stored sequentially. */ +} mld_shake128x3ctx; + +#if !defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +#define mld_shake128x3_absorb_once MLD_NAMESPACE(shake128x3_absorb_once) +MLD_INTERNAL_API +void mld_shake128x3_absorb_once(mld_shake128x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mld_shake128x3ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(state, sizeof(mld_shake128x3ctx))) +); + +#define mld_shake128x3_squeezeblocks MLD_NAMESPACE(shake128x3_squeezeblocks) +MLD_INTERNAL_API +void mld_shake128x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake128x3ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mld_shake128x3ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE128_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE128_RATE), + memory_slice(out1, nblocks * SHAKE128_RATE), + memory_slice(out2, nblocks * SHAKE128_RATE), + memory_slice(state, sizeof(mld_shake128x3ctx))) +); + +#define mld_shake128x3_init MLD_NAMESPACE(shake128x3_init) +MLD_INTERNAL_API +void mld_shake128x3_init(mld_shake128x3ctx *state); + +#define mld_shake128x3_release MLD_NAMESPACE(shake128x3_release) +MLD_INTERNAL_API +void mld_shake128x3_release(mld_shake128x3ctx *state); +#endif /* !MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +/** Context for the non-incremental 3-way SHAKE256 API. */ +typedef struct +{ + uint64_t ctx[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; /**< 3-way Keccak state, + stored sequentially. */ +} mld_shake256x3ctx; + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) || \ + (!defined(MLD_CONFIG_NO_SIGN_API) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST))) +#define mld_shake256x3_absorb_once MLD_NAMESPACE(shake256x3_absorb_once) +MLD_INTERNAL_API +void mld_shake256x3_absorb_once(mld_shake256x3ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen) +__contract__( + requires(inlen <= MLD_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mld_shake256x3ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + assigns(memory_slice(state, sizeof(mld_shake256x3ctx))) +); + +#define mld_shake256x3_squeezeblocks MLD_NAMESPACE(shake256x3_squeezeblocks) +MLD_INTERNAL_API +void mld_shake256x3_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, mld_shake256x3ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mld_shake256x3ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE256_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE256_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE256_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE256_RATE), + memory_slice(out1, nblocks * SHAKE256_RATE), + memory_slice(out2, nblocks * SHAKE256_RATE), + memory_slice(state, sizeof(mld_shake256x3ctx))) +); + +#define mld_shake256x3_init MLD_NAMESPACE(shake256x3_init) +MLD_INTERNAL_API +void mld_shake256x3_init(mld_shake256x3ctx *state); + +#define mld_shake256x3_release MLD_NAMESPACE(shake256x3_release) +MLD_INTERNAL_API +void mld_shake256x3_release(mld_shake256x3ctx *state); +#endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ + +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ +#endif /* !MLD_FIPS202_FIPS202X3_H */ diff --git a/mldsa/src/fips202/keccakf1600.c b/mldsa/src/fips202/keccakf1600.c index 18d2e6e04b..c8b0ad45bd 100644 --- a/mldsa/src/fips202/keccakf1600.c +++ b/mldsa/src/fips202/keccakf1600.c @@ -197,6 +197,95 @@ void mld_keccakf1600x4_permute(uint64_t *state) mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 2); mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 3); } + +#if defined(MLD_CONFIG_USE_X3) +/* 3-way (x3) batched Keccak primitives (see keccakf1600.h). + * xor_bytes / extract_bytes have no native x3 variant (like x4); they loop the + * serial primitive over the 3 sub-states. permute dispatches to the native x3 + * kernel when MLD_USE_NATIVE_FIPS202_X3 is defined (MLD_CONFIG_USE_X3 on + * AArch64), else falls back to 3x serial permute. */ +static void mld_keccakf1600x3_extract_bytes_c(uint64_t *state, + unsigned char *data0, + unsigned char *data1, + unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) +) +{ + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 0, data0, offset, + length); + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 1, data1, offset, + length); + mld_keccakf1600_extract_bytes(state + MLD_KECCAK_LANES * 2, data2, offset, + length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned offset, unsigned length) +{ + mld_keccakf1600x3_extract_bytes_c(state, data0, data1, data2, offset, length); +} + +static void mld_keccakf1600x3_xor_bytes_c(uint64_t *state, + const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +) +{ + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 0, data0, offset, + length); + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 1, data1, offset, + length); + mld_keccakf1600_xor_bytes(state + MLD_KECCAK_LANES * 2, data2, offset, + length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length) +{ + mld_keccakf1600x3_xor_bytes_c(state, data0, data1, data2, offset, length); +} + +MLD_INTERNAL_API +void mld_keccakf1600x3_permute(uint64_t *state) +{ +#if defined(MLD_USE_NATIVE_FIPS202_X3) + if (mld_keccak_f1600_x3_native(state) == MLD_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLD_USE_NATIVE_FIPS202_X3 */ + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 0); + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 1); + mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 2); +} +#endif /* MLD_CONFIG_USE_X3 */ #endif /* (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) && !MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/fips202/keccakf1600.h b/mldsa/src/fips202/keccakf1600.h index c0875f4a6b..baae7e8e82 100644 --- a/mldsa/src/fips202/keccakf1600.h +++ b/mldsa/src/fips202/keccakf1600.h @@ -12,6 +12,11 @@ #define MLD_KECCAK_LANES 25 #define MLD_KECCAK_WAY 4 +/* Batching width of the 3-way (x3) Keccak path used with MLD_CONFIG_USE_X3. + * Kept separate from MLD_KECCAK_WAY (=4), which the x4 API and its contracts + * depend on. */ +#define MLD_KECCAK_WAY_X3 3 + /* * WARNING: * The contents of this structure, including the placement @@ -96,6 +101,56 @@ __contract__( requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) ); + +#if defined(MLD_CONFIG_USE_X3) +/* 3-way (x3) batched Keccak primitives, used with MLD_CONFIG_USE_X3. They + * mirror the x4 trio above but operate on 3 sequential states + * (MLD_KECCAK_WAY_X3), and back the 3-way SHAKE128/SHAKE256 in fips202x3. */ +#define mld_keccakf1600x3_extract_bytes \ + MLD_NAMESPACE(keccakf1600x3_extract_bytes) +MLD_INTERNAL_API +void mld_keccakf1600x3_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) +); + +#define mld_keccakf1600x3_xor_bytes MLD_NAMESPACE(keccakf1600x3_xor_bytes) +MLD_INTERNAL_API +void mld_keccakf1600x3_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLD_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLD_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +); + +#define mld_keccakf1600x3_permute MLD_NAMESPACE(keccakf1600x3_permute) +MLD_INTERNAL_API +void mld_keccakf1600x3_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) + assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) && !MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/fips202/native/aarch64/auto.h b/mldsa/src/fips202/native/aarch64/auto.h index dffe921385..eeaae6a2d4 100644 --- a/mldsa/src/fips202/native/aarch64/auto.h +++ b/mldsa/src/fips202/native/aarch64/auto.h @@ -67,9 +67,17 @@ */ #if defined(__APPLE__) || defined(MLD_SYS_AARCH64_FAST_SHA3) #include "x2_v84a.h" -#else +#elif defined(MLD_CONFIG_USE_X3) +/* + * 3-way hybrid (two states on Neon with SHA3, one on scalar registers, single + * pass), selected with MLD_CONFIG_USE_X3. Intended for cores that execute the + * SHA3 instructions on all SIMD pipes (e.g. Neoverse V3). When + * MLD_CONFIG_USE_X3 is not set, the x4 hybrid below is used. + */ +#include "x3_v84a_scalar.h" +#else /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && MLD_CONFIG_USE_X3 */ #include "x4_v8a_v84a_scalar.h" -#endif +#endif /* !(__APPLE__ || MLD_SYS_AARCH64_FAST_SHA3) && !MLD_CONFIG_USE_X3 */ #else /* __ARM_FEATURE_SHA3 */ diff --git a/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h b/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h index ff6d963eff..b000f8b994 100644 --- a/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h +++ b/mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -66,4 +66,18 @@ __contract__( assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) ); +/* + * 3-way hybrid Keccak-f1600: states 0 and 1 on Neon (Armv8.4-A SHA3), state 2 + * on scalar registers with lazy rotations, interleaved 1:1 in a single pass. + */ +#define mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm \ + MLD_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) +); + #endif /* !MLD_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..f2575f7bd6 --- /dev/null +++ b/mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,647 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) && !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLD_ASM_NAMESPACE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) +MLD_ASM_FN_SYMBOL(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x27, x26, x7 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x29, x0, x22 + eor x26, x29, x23 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x29, x4, x5 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x29, x30, x2 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x27, x26, x0, ror #63 + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x4, x4, x27 + eor x30, x30, x17 + rax1 v25.2d, v30.2d, v28.2d + eor x30, x30, x28 + eor x29, x29, x3 + rax1 v28.2d, v28.2d, v26.2d + eor x0, x0, x30, ror #63 + rax1 v26.2d, v26.2d, v29.2d + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + rax1 v29.2d, v29.2d, v27.2d + eor x23, x23, x30 + rax1 v27.2d, v27.2d, v30.2d + str x23, [sp, #0xd0] + eor x23, x14, x15 + eor v30.16b, v0.16b, v26.16b + eor x14, x14, x0 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x23, x23, x11 + eor x15, x15, x0 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x1, x1, x27 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x23, x23, x12 + eor x23, x23, x13 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x23, x23, x26, ror #63 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x26, x13, x0 + eor x13, x28, x23 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x28, x24, x30 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x24, x16, x23 + eor x16, x21, x30 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x21, x25, x30 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x30, x19, x23 + eor x19, x20, x23 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x20, x17, x23 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x17, x12, x0 + eor x0, x2, x27 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x2, x6, x29 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x5, x5, x27 + xar v9.2d, v22.2d, v29.2d, #0x3 + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x9, x9, x29 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x8, x8, x17, ror #2 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x7, x29 + xar v21.2d, v5.2d, v26.2d, #0x1c + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x7, x2, x5, ror #47 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x5, x17, x4, ror #25 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + bcax v9.16b, v9.16b, v11.16b, v10.16b + eor x28, x7, x28, ror #57 + bcax v10.16b, v15.16b, v12.16b, v16.16b + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bcax v11.16b, v16.16b, v13.16b, v12.16b + bic x22, x22, x24, ror #56 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bcax v13.16b, v13.16b, v15.16b, v14.16b + bic x20, x27, x20, ror #48 + bcax v14.16b, v14.16b, v16.16b, v15.16b + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x11, [sp, #0x8] + bcax v17.16b, v17.16b, v19.16b, v18.16b + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + bcax v18.16b, v18.16b, v20.16b, v19.16b + mov x24, #0x1 // =1 + bcax v19.16b, v19.16b, v21.16b, v20.16b + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + bcax v20.16b, v0.16b, v22.16b, v1.16b + bic x24, x29, x1, ror #44 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + bcax v22.16b, v22.16b, v24.16b, v23.16b + eor x1, x1, x4, ror #21 + bcax v23.16b, v23.16b, v0.16b, v24.16b + ldr x11, [x11] + bic x4, x21, x30, ror #57 + bcax v24.16b, v24.16b, v1.16b, v0.16b + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bcax v0.16b, v30.16b, v2.16b, v27.16b + bic x9, x14, x6, ror #5 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x1, x1, x11 + bcax v3.16b, v3.16b, v30.16b, v4.16b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + bic x26, x19, x14, ror #41 + str x30, [sp, #0x10] + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor v0.16b, v0.16b, v28.16b + eor x6, x27, x29, ror #41 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x29, x16, x28, ror #63 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x27, x27, x12, ror #5 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x26, x30, x21, ror #26 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + rax1 v25.2d, v30.2d, v28.2d + eor x16, x30, x16 + rax1 v28.2d, v28.2d, v26.2d + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + rax1 v26.2d, v26.2d, v29.2d + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + rax1 v29.2d, v29.2d, v27.2d + eor x19, x30, x19, ror #37 + rax1 v27.2d, v27.2d, v30.2d + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x26, x26, x0, ror #55 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + xar v2.2d, v12.2d, v29.2d, #0x15 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x27, x28, x27, ror #61 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x11, x0, x11, ror #50 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x21, x26, x1 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x22, x28, x22, ror #44 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x4, x26, x4, ror #54 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x26, x2, ror #61 + xar v9.2d, v22.2d, v29.2d, #0x3 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + xar v22.2d, v14.2d, v27.2d, #0x19 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x7, x7, x22, ror #25 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + xar v24.2d, v21.2d, v25.2d, #0x3e + bic x6, x19, x13, ror #42 + xar v21.2d, v5.2d, v26.2d, #0x1c + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x5, x21, x5, ror #21 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bcax v6.16b, v11.16b, v8.16b, v7.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x10, x16, x15, ror #31 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bcax v9.16b, v9.16b, v11.16b, v10.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x17, x10, x9, ror #47 + bcax v11.16b, v16.16b, v13.16b, v12.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x20, x4, x28, ror #2 + bcax v13.16b, v13.16b, v15.16b, v14.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v14.16b, v14.16b, v16.16b, v15.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x1, x1, x11, ror #57 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v17.16b, v17.16b, v19.16b, v18.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bcax v18.16b, v18.16b, v20.16b, v19.16b + cmp x25, #0x17 + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + bcax v20.16b, v0.16b, v22.16b, v1.16b + eor x1, x5, x28 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bcax v22.16b, v22.16b, v24.16b, v23.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bcax v23.16b, v23.16b, v0.16b, v24.16b + bic x27, x24, x9, ror #47 + bcax v24.16b, v24.16b, v1.16b, v0.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x13, x13, x9, ror #57 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v3.16b, v3.16b, v30.16b, v4.16b + bic x14, x14, x8, ror #5 + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + b.le Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLD_ASM_FN_SIZE(keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c b/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c index eb578bcfcd..df0652c1f4 100644 --- a/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c +++ b/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c @@ -14,6 +14,7 @@ #if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \ defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \ defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) @@ -34,7 +35,8 @@ MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t #else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED */ @@ -42,6 +44,7 @@ MLD_EMPTY_CU(fips202_aarch64_round_constants) #endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \ MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \ - || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \ + MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h b/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h new file mode 100644 index 0000000000..fa3250a0c1 --- /dev/null +++ b/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H +#define MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X3 +/* Guard for assembly file */ +#define MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || + !mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm( + state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_FIPS202_NATIVE_AARCH64_X3_V84A_SCALAR_H */ diff --git a/mldsa/src/fips202/native/api.h b/mldsa/src/fips202/native/api.h index 7abac85a4e..36958d5ca6 100644 --- a/mldsa/src/fips202/native/api.h +++ b/mldsa/src/fips202/native/api.h @@ -65,6 +65,21 @@ __contract__( ensures((return_value == MLD_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4)) ); #endif /* MLD_USE_NATIVE_FIPS202_X4 */ +/* + * 3-fold batched Keccak-F1600 (used with MLD_CONFIG_USE_X3). To enable, set + * MLD_USE_NATIVE_FIPS202_X3 in your backend and define the inline wrapper + * mld_keccak_f1600_x3_native() to forward to your implementation. + */ +#if defined(MLD_USE_NATIVE_FIPS202_X3) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x3_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 3)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 3)) + ensures(return_value == MLD_NATIVE_FUNC_FALLBACK || return_value == MLD_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLD_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 3)) +); +#endif /* MLD_USE_NATIVE_FIPS202_X3 */ /* * Native x4 XOR bytes and extract bytes interface. diff --git a/mldsa/src/symmetric.h b/mldsa/src/symmetric.h index 4bbd4f19c0..5dfe642f36 100644 --- a/mldsa/src/symmetric.h +++ b/mldsa/src/symmetric.h @@ -11,7 +11,10 @@ #include MLD_FIPS202_HEADER_FILE #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) #include MLD_FIPS202X4_HEADER_FILE +#if defined(MLD_CONFIG_USE_X3) +#include "fips202/fips202x3.h" #endif +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ #define MLD_STREAM128_BLOCKBYTES SHAKE128_RATE #define MLD_STREAM256_BLOCKBYTES SHAKE256_RATE @@ -65,4 +68,28 @@ (NBLOCKS), (CTX)) #define mld_xof128_x4_release(CTX) mld_shake128x4_release((CTX)) +#if defined(MLD_CONFIG_USE_X3) +/* + * 3-way XOF, used with MLD_CONFIG_USE_X3. Mirrors the x4 macros above but + * 3-wide. SHAKE128 backs the x3 ExpandA path (mld_poly_uniform_3x); SHAKE256 + * backs the x3 ExpandS and ExpandMask paths (mld_poly_uniform_eta_3x, + * mld_poly_uniform_gamma1_3x). + */ +#define mld_xof128_x3_ctx mld_shake128x3ctx +#define mld_xof128_x3_init(CTX) mld_shake128x3_init((CTX)) +#define mld_xof128_x3_absorb(CTX, IN, INBYTES) \ + mld_shake128x3_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (INBYTES)) +#define mld_xof128_x3_squeezeblocks(BUF, NBLOCKS, CTX) \ + mld_shake128x3_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (NBLOCKS), (CTX)) +#define mld_xof128_x3_release(CTX) mld_shake128x3_release((CTX)) + +#define mld_xof256_x3_ctx mld_shake256x3ctx +#define mld_xof256_x3_init(CTX) mld_shake256x3_init((CTX)) +#define mld_xof256_x3_absorb(CTX, IN, INBYTES) \ + mld_shake256x3_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (INBYTES)) +#define mld_xof256_x3_squeezeblocks(BUF, NBLOCKS, CTX) \ + mld_shake256x3_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (NBLOCKS), (CTX)) +#define mld_xof256_x3_release(CTX) mld_shake256x3_release((CTX)) +#endif /* MLD_CONFIG_USE_X3 */ + #endif /* !MLD_SYMMETRIC_H */ diff --git a/proofs/cbmc/mldsa_native_config_cbmc.h b/proofs/cbmc/mldsa_native_config_cbmc.h index 8c5e3cf7f5..316aef6f16 100644 --- a/proofs/cbmc/mldsa_native_config_cbmc.h +++ b/proofs/cbmc/mldsa_native_config_cbmc.h @@ -414,6 +414,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/scripts/autogen b/scripts/autogen index 6cc399fe33..6ea7f7d48e 100755 --- a/scripts/autogen +++ b/scripts/autogen @@ -1588,6 +1588,7 @@ def gen_aarch64_keccak_constants_c_file(): yield "#if (defined(MLD_FIPS202_AARCH64_NEED_X1_SCALAR) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X1_V84A) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X2_V84A) || \\" + yield " defined(MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \\" yield " defined(MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \\" yield " !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)" @@ -1602,7 +1603,8 @@ def gen_aarch64_keccak_constants_c_file(): yield "" yield "#else /* (MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \\" yield " MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \\" - yield " || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" + yield " || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \\" + yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \\" yield " !MLD_CONFIG_MULTILEVEL_NO_SHARED */" yield "" @@ -1610,7 +1612,8 @@ def gen_aarch64_keccak_constants_c_file(): yield "" yield "#endif /* !((MLD_FIPS202_AARCH64_NEED_X1_SCALAR || \\" yield " MLD_FIPS202_AARCH64_NEED_X1_V84A || MLD_FIPS202_AARCH64_NEED_X2_V84A \\" - yield " || MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" + yield " || MLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID || \\" + yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \\" yield " MLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \\" yield " !MLD_CONFIG_MULTILEVEL_NO_SHARED) */" yield "" @@ -5218,6 +5221,7 @@ def _main(): "keccak_f1600_x1_scalar_aarch64_asm", "keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm", "keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm", + "keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm", ] parser = argparse.ArgumentParser( diff --git a/test/configs/break_pct_config.h b/test/configs/break_pct_config.h index e435417682..570de15fc8 100644 --- a/test/configs/break_pct_config.h +++ b/test/configs/break_pct_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_heap_alloc_config.h b/test/configs/custom_heap_alloc_config.h index b2201f3207..50f63f8f59 100644 --- a/test/configs/custom_heap_alloc_config.h +++ b/test/configs/custom_heap_alloc_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_memcpy_config.h b/test/configs/custom_memcpy_config.h index b2728231ba..6c0d06b106 100644 --- a/test/configs/custom_memcpy_config.h +++ b/test/configs/custom_memcpy_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_memset_config.h b/test/configs/custom_memset_config.h index f0d5b7b214..91cf51bc55 100644 --- a/test/configs/custom_memset_config.h +++ b/test/configs/custom_memset_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_0.h b/test/configs/custom_native_capability_config_0.h index 4cc7e99437..dac6e7863c 100644 --- a/test/configs/custom_native_capability_config_0.h +++ b/test/configs/custom_native_capability_config_0.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_1.h b/test/configs/custom_native_capability_config_1.h index 281f910865..62194dd1bf 100644 --- a/test/configs/custom_native_capability_config_1.h +++ b/test/configs/custom_native_capability_config_1.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_CPUID_AVX2.h b/test/configs/custom_native_capability_config_CPUID_AVX2.h index 6e36b64571..d38a143c19 100644 --- a/test/configs/custom_native_capability_config_CPUID_AVX2.h +++ b/test/configs/custom_native_capability_config_CPUID_AVX2.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h b/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h index 760c7b7492..6a0c084b7f 100644 --- a/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h +++ b/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_randombytes_config.h b/test/configs/custom_randombytes_config.h index 60ee5bc69d..a2424bd9a9 100644 --- a/test/configs/custom_randombytes_config.h +++ b/test/configs/custom_randombytes_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_stdlib_config.h b/test/configs/custom_stdlib_config.h index b4132acc61..b1d76007ed 100644 --- a/test/configs/custom_stdlib_config.h +++ b/test/configs/custom_stdlib_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/custom_zeroize_config.h b/test/configs/custom_zeroize_config.h index 249e22aa86..cbf8ab66f1 100644 --- a/test/configs/custom_zeroize_config.h +++ b/test/configs/custom_zeroize_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/low_signing_bound_config.h b/test/configs/low_signing_bound_config.h index f238b4a25f..9262dc76af 100644 --- a/test/configs/low_signing_bound_config.h +++ b/test/configs/low_signing_bound_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/no_asm_config.h b/test/configs/no_asm_config.h index 03a8dee9e6..ccb07fa695 100644 --- a/test/configs/no_asm_config.h +++ b/test/configs/no_asm_config.h @@ -411,6 +411,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/serial_fips202_config.h b/test/configs/serial_fips202_config.h index 413495299e..5ab09bcc67 100644 --- a/test/configs/serial_fips202_config.h +++ b/test/configs/serial_fips202_config.h @@ -410,6 +410,29 @@ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/test_alloc_config.h b/test/configs/test_alloc_config.h index 441e16682c..7b443121d7 100644 --- a/test/configs/test_alloc_config.h +++ b/test/configs/test_alloc_config.h @@ -417,6 +417,29 @@ struct test_ctx_t; /* Forward declaration */ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * diff --git a/test/configs/test_sign_hook_config.h b/test/configs/test_sign_hook_config.h index 90f163ca3c..7a8a1b8e6f 100644 --- a/test/configs/test_sign_hook_config.h +++ b/test/configs/test_sign_hook_config.h @@ -421,6 +421,29 @@ struct test_sign_hook_ctx; /* Forward declaration */ /* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #endif +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#if !defined(MLD_CONFIG_USE_X3) +/* #define MLD_CONFIG_USE_X3 */ +#endif + /** * MLD_CONFIG_FIPS202_BACKEND_FILE * From 9905ad3e5c76ce371946f13954a059406845f307 Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:14 +0000 Subject: [PATCH 3/8] HOL-Light: Prove the x3 hybrid Keccak-f1600 kernel Prove functional correctness, ABI compliance, memory safety and constant-time behaviour of the x3 kernel, following the x4 v84a proof. Signed-off-by: Soumya Bhattacharya --- .github/workflows/hol_light.yml | 2 + proofs/hol_light/README.md | 1 + proofs/hol_light/aarch64/Makefile | 1 + ..._f1600_x3_v84a_scalar_hybrid_aarch64_asm.S | 642 +++++++++++++ ...f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml | 882 ++++++++++++++++++ .../aarch64/proofs/subroutine_signatures.ml | 17 + scripts/autogen | 6 + test/abicheck/aarch64/abicheck_aarch64.mk | 2 + ..._f1600_x3_v84a_scalar_hybrid_aarch64_asm.c | 86 ++ test/abicheck/aarch64/checks_aarch64_all.h | 3 + test/mk/components.mk | 1 + 11 files changed, 1643 insertions(+) create mode 100644 proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml create mode 100644 test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c diff --git a/.github/workflows/hol_light.yml b/.github/workflows/hol_light.yml index c364013574..eca809b169 100644 --- a/.github/workflows/hol_light.yml +++ b/.github/workflows/hol_light.yml @@ -110,6 +110,8 @@ jobs: needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x2_v84a_aarch64_asm needs: ["keccak_spec.ml", "keccak_constants.ml", "subroutine_signatures.ml"] + - name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm needs: ["keccak_spec.ml", "keccak_utils.ml", "keccak_constants.ml", "subroutine_signatures.ml"] - name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm diff --git a/proofs/hol_light/README.md b/proofs/hol_light/README.md index 6f0edfc4f4..42ebb6578a 100644 --- a/proofs/hol_light/README.md +++ b/proofs/hol_light/README.md @@ -219,6 +219,7 @@ All routines listed below have been proven correct, memory-safe, and secret-inde * Keccak-F1600 using lazy rotations[^HYBRID]: [keccak_f1600_x1_scalar_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x1_scalar_aarch64_asm.S) * Keccak-F1600 using v8.4-A SHA3 instructions: [keccak_f1600_x1_v84a_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x1_v84a_aarch64_asm.S) * 2-fold Keccak-F1600 using v8.4-A SHA3 instructions: [keccak_f1600_x2_v84a_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x2_v84a_aarch64_asm.S) + * 'Hybrid' 3-fold Keccak-F1600 using scalar and v8.4-A+SHA3 Neon instructions: [keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S) * 'Hybrid' 4-fold Keccak-F1600 using scalar and v8-A Neon instructions: [keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S) * 'Triple hybrid' 4-fold Keccak-F1600 using scalar, v8-A Neon and v8.4-A+SHA3 Neon instructions: [keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S](aarch64/mldsa/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S) diff --git a/proofs/hol_light/aarch64/Makefile b/proofs/hol_light/aarch64/Makefile index b4a5254618..1af35214b8 100644 --- a/proofs/hol_light/aarch64/Makefile +++ b/proofs/hol_light/aarch64/Makefile @@ -68,6 +68,7 @@ OBJ = mldsa/mldsa_intt_aarch64_asm.o \ mldsa/keccak_f1600_x1_scalar_aarch64_asm.o \ mldsa/keccak_f1600_x1_v84a_aarch64_asm.o \ mldsa/keccak_f1600_x2_v84a_aarch64_asm.o \ + mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o \ mldsa/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.o \ mldsa/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.o \ mldsa/mldsa_polyz_unpack_17_aarch64_asm.o \ diff --git a/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S b/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..8d206ae486 --- /dev/null +++ b/proofs/hol_light/aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,642 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Soumya Bhattacharya +// Derived from keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S +// (x4: 2 states on Neon + 2 on scalar; x3: 2 on Neon + 1 on scalar). +// x3 (three-state) hybrid batched Keccak-f1600: +// - 2 states processed on Neon (SHA3 extension), lanes packed 2-per-128-bit +// vector register, exactly as in the x4 kernel. +// - 1 state processed on scalar GPRs with lazy rotations. +// - Single pass: both the Neon (2 states) and the scalar (1 state) complete a +// full 24-round Keccak-f1600 within one pass. The interleave ratio is 1:1 +// (one scalar round + one vector round per loop iteration, 24 iterations), +// in contrast to the x4 kernel's 2:1 ratio + two-pass outer loop. + +// This is the SLOTHY-scheduled (interleave-only) production file. SLOTHY +// interleaved the scalar and Neon instruction streams of the symbolic source +// dev/fips202/aarch64_symbolic/keccak_f1600_x3_v84a_scalar_hybrid_clean.S, +// keeping its register allocation and the instruction order within each +// stream. + +/*yaml + Name: keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for three sequential states with ARMv8.4-A optimizations + Signature: void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 600 + permissions: read/write + c_parameter: uint64_t state[75] + description: Three sequential Keccak states (state0[25], state1[25], state2[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +#ifdef __APPLE__ +.global _mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +_mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: +#else +.global mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: +#endif + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x27, x26, x7 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x29, x0, x22 + eor x26, x29, x23 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x29, x4, x5 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x29, x30, x2 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x27, x26, x0, ror #63 + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x4, x4, x27 + eor x30, x30, x17 + rax1 v25.2d, v30.2d, v28.2d + eor x30, x30, x28 + eor x29, x29, x3 + rax1 v28.2d, v28.2d, v26.2d + eor x0, x0, x30, ror #63 + rax1 v26.2d, v26.2d, v29.2d + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + rax1 v29.2d, v29.2d, v27.2d + eor x23, x23, x30 + rax1 v27.2d, v27.2d, v30.2d + str x23, [sp, #0xd0] + eor x23, x14, x15 + eor v30.16b, v0.16b, v26.16b + eor x14, x14, x0 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x23, x23, x11 + eor x15, x15, x0 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x1, x1, x27 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x23, x23, x12 + eor x23, x23, x13 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x23, x23, x26, ror #63 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x26, x13, x0 + eor x13, x28, x23 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x28, x24, x30 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x24, x16, x23 + eor x16, x21, x30 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x21, x25, x30 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x30, x19, x23 + eor x19, x20, x23 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x20, x17, x23 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x17, x12, x0 + eor x0, x2, x27 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x2, x6, x29 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x5, x5, x27 + xar v9.2d, v22.2d, v29.2d, #0x3 + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x9, x9, x29 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x8, x8, x17, ror #2 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x7, x29 + xar v21.2d, v5.2d, v26.2d, #0x1c + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x7, x2, x5, ror #47 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x5, x17, x4, ror #25 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + bcax v9.16b, v9.16b, v11.16b, v10.16b + eor x28, x7, x28, ror #57 + bcax v10.16b, v15.16b, v12.16b, v16.16b + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bcax v11.16b, v16.16b, v13.16b, v12.16b + bic x22, x22, x24, ror #56 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bcax v13.16b, v13.16b, v15.16b, v14.16b + bic x20, x27, x20, ror #48 + bcax v14.16b, v14.16b, v16.16b, v15.16b + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x11, [sp, #0x8] + bcax v17.16b, v17.16b, v19.16b, v18.16b + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + bcax v18.16b, v18.16b, v20.16b, v19.16b + mov x24, #0x1 // =1 + bcax v19.16b, v19.16b, v21.16b, v20.16b + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + bcax v20.16b, v0.16b, v22.16b, v1.16b + bic x24, x29, x1, ror #44 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + bcax v22.16b, v22.16b, v24.16b, v23.16b + eor x1, x1, x4, ror #21 + bcax v23.16b, v23.16b, v0.16b, v24.16b + ldr x11, [x11] + bic x4, x21, x30, ror #57 + bcax v24.16b, v24.16b, v1.16b, v0.16b + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bcax v0.16b, v30.16b, v2.16b, v27.16b + bic x9, x14, x6, ror #5 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x1, x1, x11 + bcax v3.16b, v3.16b, v30.16b, v4.16b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + bic x26, x19, x14, ror #41 + str x30, [sp, #0x10] + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor v0.16b, v0.16b, v28.16b + eor x6, x27, x29, ror #41 + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x29, x16, x28, ror #63 + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x27, x27, x12, ror #5 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor x26, x30, x21, ror #26 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor3 v26.16b, v26.16b, v19.16b, v24.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + rax1 v25.2d, v30.2d, v28.2d + eor x16, x30, x16 + rax1 v28.2d, v28.2d, v26.2d + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + rax1 v26.2d, v26.2d, v29.2d + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + rax1 v29.2d, v29.2d, v27.2d + eor x19, x30, x19, ror #37 + rax1 v27.2d, v27.2d, v30.2d + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x26, x26, x0, ror #55 + xar v0.2d, v2.2d, v29.2d, #0x2 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + xar v2.2d, v12.2d, v29.2d, #0x15 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + xar v12.2d, v13.2d, v28.2d, #0x27 + eor x27, x28, x27, ror #61 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + xar v19.2d, v23.2d, v28.2d, #0x8 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x11, x0, x11, ror #50 + xar v15.2d, v1.2d, v25.2d, #0x3f + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x21, x26, x1 + xar v8.2d, v16.2d, v25.2d, #0x13 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + xar v16.2d, v7.2d, v29.2d, #0x3a + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + xar v7.2d, v10.2d, v26.2d, #0x3d + eor x22, x28, x22, ror #44 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + xar v3.2d, v18.2d, v28.2d, #0x2b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + eor x4, x26, x4, ror #54 + xar v17.2d, v11.2d, v25.2d, #0x36 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x26, x2, ror #61 + xar v9.2d, v22.2d, v29.2d, #0x3 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + xar v22.2d, v14.2d, v27.2d, #0x19 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + xar v14.2d, v20.2d, v26.2d, #0x2e + eor x7, x7, x22, ror #25 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + xar v4.2d, v24.2d, v27.2d, #0x32 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + xar v24.2d, v21.2d, v25.2d, #0x3e + bic x6, x19, x13, ror #42 + xar v21.2d, v5.2d, v26.2d, #0x1c + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x5, x21, x5, ror #21 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bcax v6.16b, v11.16b, v8.16b, v7.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v7.16b, v7.16b, v9.16b, v8.16b + bic x10, x16, x15, ror #31 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bcax v9.16b, v9.16b, v11.16b, v10.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x17, x10, x9, ror #47 + bcax v11.16b, v16.16b, v13.16b, v12.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bcax v12.16b, v12.16b, v14.16b, v13.16b + bic x20, x4, x28, ror #2 + bcax v13.16b, v13.16b, v15.16b, v14.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v14.16b, v14.16b, v16.16b, v15.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x1, x1, x11, ror #57 + bcax v16.16b, v21.16b, v18.16b, v17.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v17.16b, v17.16b, v19.16b, v18.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bcax v18.16b, v18.16b, v20.16b, v19.16b + cmp x25, #0x17 + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + bcax v20.16b, v0.16b, v22.16b, v1.16b + eor x1, x5, x28 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bcax v22.16b, v22.16b, v24.16b, v23.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bcax v23.16b, v23.16b, v0.16b, v24.16b + bic x27, x24, x9, ror #47 + bcax v24.16b, v24.16b, v1.16b, v0.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + bcax v1.16b, v27.16b, v3.16b, v2.16b + eor x13, x13, x9, ror #57 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v3.16b, v3.16b, v30.16b, v4.16b + bic x14, x14, x8, ror #5 + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop_end: + b.le Lmld_keccak_f1600_x3_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml b/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml new file mode 100644 index 0000000000..84e46d9df7 --- /dev/null +++ b/proofs/hol_light/aarch64/proofs/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.ml @@ -0,0 +1,882 @@ +(* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0 + *) + +(* ========================================================================= *) +(* 3-way batch Keccak-f1600 scalar/vector hybrid code (single pass). *) +(* Derived from the x4 v84a proof: *) +(* keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml *) +(* ========================================================================= *) + +(* Load base theories for AArch64 from s2n-bignum *) +needs "s2n_bignum/arm/proofs/base.ml";; + +needs "mldsa_native/aarch64/proofs/keccak_utils.ml";; + +(**** print_literal_from_elf "aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o";; + ****) + +let keccak_f1600_x3_v84a_scalar_mc = define_assert_from_elf + "keccak_f1600_x3_v84a_scalar_mc" "aarch64/mldsa/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.o" +(*** BYTECODE START ***) +[ + 0xd10383ff; (* arm_SUB SP SP (rvalue (word 224)) *) + 0xa90353f3; (* arm_STP X19 X20 SP (Immediate_Offset (iword (&48))) *) + 0xa9045bf5; (* arm_STP X21 X22 SP (Immediate_Offset (iword (&64))) *) + 0xa90563f7; (* arm_STP X23 X24 SP (Immediate_Offset (iword (&80))) *) + 0xa9066bf9; (* arm_STP X25 X26 SP (Immediate_Offset (iword (&96))) *) + 0xa90773fb; (* arm_STP X27 X28 SP (Immediate_Offset (iword (&112))) *) + 0xa9087bfd; (* arm_STP X29 X30 SP (Immediate_Offset (iword (&128))) *) + 0x6d0927e8; (* arm_STP D8 D9 SP (Immediate_Offset (iword (&144))) *) + 0x6d0a2fea; (* arm_STP D10 D11 SP (Immediate_Offset (iword (&160))) *) + 0x6d0b37ec; (* arm_STP D12 D13 SP (Immediate_Offset (iword (&176))) *) + 0x6d0c3fee; (* arm_STP D14 D15 SP (Immediate_Offset (iword (&192))) *) + 0xaa0103fd; (* arm_MOV X29 X1 *) + 0xf90007fd; (* arm_STR X29 SP (Immediate_Offset (word 8)) *) + 0xf9000bfd; (* arm_STR X29 SP (Immediate_Offset (word 16)) *) + 0xf90003e0; (* arm_STR X0 SP (Immediate_Offset (word 0)) *) + 0x91032004; (* arm_ADD X4 X0 (rvalue (word 200)) *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b20; (* arm_TRN1 Q0 Q25 Q27 64 128 *) + 0x4edb6b21; (* arm_TRN2 Q1 Q25 Q27 64 128 *) + 0x4edc2b42; (* arm_TRN1 Q2 Q26 Q28 64 128 *) + 0x4edc6b43; (* arm_TRN2 Q3 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b24; (* arm_TRN1 Q4 Q25 Q27 64 128 *) + 0x4edb6b25; (* arm_TRN2 Q5 Q25 Q27 64 128 *) + 0x4edc2b46; (* arm_TRN1 Q6 Q26 Q28 64 128 *) + 0x4edc6b47; (* arm_TRN2 Q7 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b28; (* arm_TRN1 Q8 Q25 Q27 64 128 *) + 0x4edb6b29; (* arm_TRN2 Q9 Q25 Q27 64 128 *) + 0x4edc2b4a; (* arm_TRN1 Q10 Q26 Q28 64 128 *) + 0x4edc6b4b; (* arm_TRN2 Q11 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b2c; (* arm_TRN1 Q12 Q25 Q27 64 128 *) + 0x4edb6b2d; (* arm_TRN2 Q13 Q25 Q27 64 128 *) + 0x4edc2b4e; (* arm_TRN1 Q14 Q26 Q28 64 128 *) + 0x4edc6b4f; (* arm_TRN2 Q15 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b30; (* arm_TRN1 Q16 Q25 Q27 64 128 *) + 0x4edb6b31; (* arm_TRN2 Q17 Q25 Q27 64 128 *) + 0x4edc2b52; (* arm_TRN1 Q18 Q26 Q28 64 128 *) + 0x4edc6b53; (* arm_TRN2 Q19 Q26 Q28 64 128 *) + 0xacc16819; (* arm_LDP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4cdfac9b; (* arm_LDP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4edb2b34; (* arm_TRN1 Q20 Q25 Q27 64 128 *) + 0x4edb6b35; (* arm_TRN2 Q21 Q25 Q27 64 128 *) + 0x4edc2b56; (* arm_TRN1 Q22 Q26 Q28 64 128 *) + 0x4edc6b57; (* arm_TRN2 Q23 Q26 Q28 64 128 *) + 0xfd400019; (* arm_LDR D25 X0 (Immediate_Offset (word 0)) *) + 0xfd40009b; (* arm_LDR D27 X4 (Immediate_Offset (word 0)) *) + 0x4edb2b38; (* arm_TRN1 Q24 Q25 Q27 64 128 *) + 0xd1030000; (* arm_SUB X0 X0 (rvalue (word 192)) *) + 0x91064000; (* arm_ADD X0 X0 (rvalue (word 400)) *) + 0xa9401801; (* arm_LDP X1 X6 X0 (Immediate_Offset (iword (&0))) *) + 0xa941400b; (* arm_LDP X11 X16 X0 (Immediate_Offset (iword (&16))) *) + 0xa9420815; (* arm_LDP X21 X2 X0 (Immediate_Offset (iword (&32))) *) + 0xa9433007; (* arm_LDP X7 X12 X0 (Immediate_Offset (iword (&48))) *) + 0xa9445811; (* arm_LDP X17 X22 X0 (Immediate_Offset (iword (&64))) *) + 0xa9452003; (* arm_LDP X3 X8 X0 (Immediate_Offset (iword (&80))) *) + 0xa946700d; (* arm_LDP X13 X28 X0 (Immediate_Offset (iword (&96))) *) + 0xa9471017; (* arm_LDP X23 X4 X0 (Immediate_Offset (iword (&112))) *) + 0xa9483809; (* arm_LDP X9 X14 X0 (Immediate_Offset (iword (&128))) *) + 0xa9496013; (* arm_LDP X19 X24 X0 (Immediate_Offset (iword (&144))) *) + 0xa94a2805; (* arm_LDP X5 X10 X0 (Immediate_Offset (iword (&160))) *) + 0xa94b500f; (* arm_LDP X15 X20 X0 (Immediate_Offset (iword (&176))) *) + 0xf9406019; (* arm_LDR X25 X0 (Immediate_Offset (word 192)) *) + 0xd1064000; (* arm_SUB X0 X0 (rvalue (word 400)) *) + 0xca19031e; (* arm_EOR X30 X24 X25 *) + 0xca0a013b; (* arm_EOR X27 X9 X10 *) + 0xce05281e; (* arm_EOR3 Q30 Q0 Q5 Q10 *) + 0xce0f53de; (* arm_EOR3 Q30 Q30 Q15 Q20 *) + 0xca1503c0; (* arm_EOR X0 X30 X21 *) + 0xca06037a; (* arm_EOR X26 X27 X6 *) + 0xce062c3d; (* arm_EOR3 Q29 Q1 Q6 Q11 *) + 0xca07035b; (* arm_EOR X27 X26 X7 *) + 0xce1057bd; (* arm_EOR3 Q29 Q29 Q16 Q21 *) + 0xca16001d; (* arm_EOR X29 X0 X22 *) + 0xca1703ba; (* arm_EOR X26 X29 X23 *) + 0xce07305c; (* arm_EOR3 Q28 Q2 Q7 Q12 *) + 0xca05009d; (* arm_EOR X29 X4 X5 *) + 0xce115b9c; (* arm_EOR3 Q28 Q28 Q17 Q22 *) + 0xca0103be; (* arm_EOR X30 X29 X1 *) + 0xca080360; (* arm_EOR X0 X27 X8 *) + 0xce08347b; (* arm_EOR3 Q27 Q3 Q8 Q13 *) + 0xca0203dd; (* arm_EOR X29 X30 X2 *) + 0xce125f7b; (* arm_EOR3 Q27 Q27 Q18 Q23 *) + 0xca14027e; (* arm_EOR X30 X19 X20 *) + 0xca1003de; (* arm_EOR X30 X30 X16 *) + 0xce09389a; (* arm_EOR3 Q26 Q4 Q9 Q14 *) + 0xcac0ff5b; (* arm_EOR X27 X26 (Shiftedreg X0 ROR 63) *) + 0xce13635a; (* arm_EOR3 Q26 Q26 Q19 Q24 *) + 0xca1b0084; (* arm_EOR X4 X4 X27 *) + 0xca1103de; (* arm_EOR X30 X30 X17 *) + 0xce7c8fd9; (* arm_RAX1 Q25 Q30 Q28 *) + 0xca1c03de; (* arm_EOR X30 X30 X28 *) + 0xca0303bd; (* arm_EOR X29 X29 X3 *) + 0xce7a8f9c; (* arm_RAX1 Q28 Q28 Q26 *) + 0xcadefc00; (* arm_EOR X0 X0 (Shiftedreg X30 ROR 63) *) + 0xce7d8f5a; (* arm_RAX1 Q26 Q26 Q29 *) + 0xcaddffde; (* arm_EOR X30 X30 (Shiftedreg X29 ROR 63) *) + 0xca1e02d6; (* arm_EOR X22 X22 X30 *) + 0xce7b8fbd; (* arm_RAX1 Q29 Q29 Q27 *) + 0xca1e02f7; (* arm_EOR X23 X23 X30 *) + 0xce7e8f7b; (* arm_RAX1 Q27 Q27 Q30 *) + 0xf9006bf7; (* arm_STR X23 SP (Immediate_Offset (word 208)) *) + 0xca0f01d7; (* arm_EOR X23 X14 X15 *) + 0x6e3a1c1e; (* arm_EOR_VEC Q30 Q0 Q26 128 *) + 0xca0001ce; (* arm_EOR X14 X14 X0 *) + 0xce9d0840; (* arm_XAR Q0 Q2 Q29 (word 2) *) + 0xca0b02f7; (* arm_EOR X23 X23 X11 *) + 0xca0001ef; (* arm_EOR X15 X15 X0 *) + 0xce9d5582; (* arm_XAR Q2 Q12 Q29 (word 21) *) + 0xca1b0021; (* arm_EOR X1 X1 X27 *) + 0xce9c9dac; (* arm_XAR Q12 Q13 Q28 (word 39) *) + 0xca0c02f7; (* arm_EOR X23 X23 X12 *) + 0xca0d02f7; (* arm_EOR X23 X23 X13 *) + 0xce9be26d; (* arm_XAR Q13 Q19 Q27 (word 56) *) + 0xca00016b; (* arm_EOR X11 X11 X0 *) + 0xcad7ffbd; (* arm_EOR X29 X29 (Shiftedreg X23 ROR 63) *) + 0xce9c22f3; (* arm_XAR Q19 Q23 Q28 (word 8) *) + 0xcadafef7; (* arm_EOR X23 X23 (Shiftedreg X26 ROR 63) *) + 0xce9a5df7; (* arm_XAR Q23 Q15 Q26 (word 23) *) + 0xca0001ba; (* arm_EOR X26 X13 X0 *) + 0xca17038d; (* arm_EOR X13 X28 X23 *) + 0xce99fc2f; (* arm_XAR Q15 Q1 Q25 (word 63) *) + 0xca1e031c; (* arm_EOR X28 X24 X30 *) + 0xce9c2501; (* arm_XAR Q1 Q8 Q28 (word 9) *) + 0xca170218; (* arm_EOR X24 X16 X23 *) + 0xca1e02b0; (* arm_EOR X16 X21 X30 *) + 0xce994e08; (* arm_XAR Q8 Q16 Q25 (word 19) *) + 0xca1e0335; (* arm_EOR X21 X25 X30 *) + 0xce9de8f0; (* arm_XAR Q16 Q7 Q29 (word 58) *) + 0xca17027e; (* arm_EOR X30 X19 X23 *) + 0xca170293; (* arm_EOR X19 X20 X23 *) + 0xce9af547; (* arm_XAR Q7 Q10 Q26 (word 61) *) + 0xca170234; (* arm_EOR X20 X17 X23 *) + 0xce9c906a; (* arm_XAR Q10 Q3 Q28 (word 36) *) + 0xca000191; (* arm_EOR X17 X12 X0 *) + 0xca1b0040; (* arm_EOR X0 X2 X27 *) + 0xce9cae43; (* arm_XAR Q3 Q18 Q28 (word 43) *) + 0xca1d00c2; (* arm_EOR X2 X6 X29 *) + 0xce9dc632; (* arm_XAR Q18 Q17 Q29 (word 49) *) + 0xca1d0106; (* arm_EOR X6 X8 X29 *) + 0x8aedbf88; (* arm_BIC X8 X28 (Shiftedreg X13 ROR 47) *) + 0xce99d971; (* arm_XAR Q17 Q11 Q25 (word 54) *) + 0xca1b006c; (* arm_EOR X12 X3 X27 *) + 0x8af14da3; (* arm_BIC X3 X13 (Shiftedreg X17 ROR 19) *) + 0xce9bb12b; (* arm_XAR Q11 Q9 Q27 (word 44) *) + 0xca1b00a5; (* arm_EOR X5 X5 X27 *) + 0xce9d0ec9; (* arm_XAR Q9 Q22 Q29 (word 3) *) + 0xf9406bfb; (* arm_LDR X27 SP (Immediate_Offset (word 208)) *) + 0x8ae21639; (* arm_BIC X25 X17 (Shiftedreg X2 ROR 5) *) + 0xce9b65d6; (* arm_XAR Q22 Q14 Q27 (word 25) *) + 0xca1d0129; (* arm_EOR X9 X9 X29 *) + 0xce9aba8e; (* arm_XAR Q14 Q20 Q26 (word 46) *) + 0xcac5d337; (* arm_EOR X23 X25 (Shiftedreg X5 ROR 52) *) + 0xcac26063; (* arm_EOR X3 X3 (Shiftedreg X2 ROR 24) *) + 0xce9b9494; (* arm_XAR Q20 Q4 Q27 (word 37) *) + 0xcad10908; (* arm_EOR X8 X8 (Shiftedreg X17 ROR 2) *) + 0xce9bcb04; (* arm_XAR Q4 Q24 Q27 (word 50) *) + 0xca1d0151; (* arm_EOR X17 X10 X29 *) + 0x8af6bd99; (* arm_BIC X25 X12 (Shiftedreg X22 ROR 47) *) + 0xce99fab8; (* arm_XAR Q24 Q21 Q25 (word 62) *) + 0xca1d00fd; (* arm_EOR X29 X7 X29 *) + 0xce9a70b5; (* arm_XAR Q21 Q5 Q26 (word 28) *) + 0x8afb088a; (* arm_BIC X10 X4 (Shiftedreg X27 ROR 2) *) + 0x8afc28a7; (* arm_BIC X7 X5 (Shiftedreg X28 ROR 10) *) + 0xce9950db; (* arm_XAR Q27 Q6 Q25 (word 20) *) + 0xcad4c94a; (* arm_EOR X10 X10 (Shiftedreg X20 ROR 50) *) + 0xcacde4ed; (* arm_EOR X13 X7 (Shiftedreg X13 ROR 57) *) + 0xce272d45; (* arm_BCAX Q5 Q10 Q7 Q11 *) + 0x8ae5bc47; (* arm_BIC X7 X2 (Shiftedreg X5 ROR 47) *) + 0xce281d66; (* arm_BCAX Q6 Q11 Q8 Q7 *) + 0xcad89f22; (* arm_EOR X2 X25 (Shiftedreg X24 ROR 39) *) + 0x8aebe699; (* arm_BIC X25 X20 (Shiftedreg X11 ROR 57) *) + 0xce2920e7; (* arm_BCAX Q7 Q7 Q9 Q8 *) + 0x8ae46625; (* arm_BIC X5 X17 (Shiftedreg X4 ROR 25) *) + 0xce2a2508; (* arm_BCAX Q8 Q8 Q10 Q9 *) + 0xcad1d739; (* arm_EOR X25 X25 (Shiftedreg X17 ROR 53) *) + 0x8af1f171; (* arm_BIC X17 X11 (Shiftedreg X17 ROR 60) *) + 0xce2b2929; (* arm_BCAX Q9 Q9 Q11 Q10 *) + 0xcadce4fc; (* arm_EOR X28 X7 (Shiftedreg X28 ROR 57) *) + 0xce2c41ea; (* arm_BCAX Q10 Q15 Q12 Q16 *) + 0x8aeca927; (* arm_BIC X7 X9 (Shiftedreg X12 ROR 42) *) + 0xcad664e7; (* arm_EOR X7 X7 (Shiftedreg X22 ROR 25) *) + 0xce2d320b; (* arm_BCAX Q11 Q16 Q13 Q12 *) + 0x8af8e2d6; (* arm_BIC X22 X22 (Shiftedreg X24 ROR 56) *) + 0xce2e358c; (* arm_BCAX Q12 Q12 Q14 Q13 *) + 0x8aef7f18; (* arm_BIC X24 X24 (Shiftedreg X15 ROR 31) *) + 0xcacf5ed6; (* arm_EOR X22 X22 (Shiftedreg X15 ROR 23) *) + 0xce2f39ad; (* arm_BCAX Q13 Q13 Q15 Q14 *) + 0x8af4c374; (* arm_BIC X20 X27 (Shiftedreg X20 ROR 48) *) + 0xce303dce; (* arm_BCAX Q14 Q14 Q16 Q15 *) + 0x8ae941ef; (* arm_BIC X15 X15 (Shiftedreg X9 ROR 16) *) + 0xcacce9ec; (* arm_EOR X12 X15 (Shiftedreg X12 ROR 58) *) + 0xce31568f; (* arm_BCAX Q15 Q20 Q17 Q21 *) + 0xcadb6caf; (* arm_EOR X15 X5 (Shiftedreg X27 ROR 27) *) + 0xcacba685; (* arm_EOR X5 X20 (Shiftedreg X11 ROR 41) *) + 0xce3246b0; (* arm_BCAX Q16 Q21 Q18 Q17 *) + 0xf94007eb; (* arm_LDR X11 SP (Immediate_Offset (word 8)) *) + 0xce334a31; (* arm_BCAX Q17 Q17 Q19 Q18 *) + 0xcac45634; (* arm_EOR X20 X17 (Shiftedreg X4 ROR 21) *) + 0xcac9bf11; (* arm_EOR X17 X24 (Shiftedreg X9 ROR 47) *) + 0xce344e52; (* arm_BCAX Q18 Q18 Q20 Q19 *) + 0xd2800038; (* arm_MOV X24 (rvalue (word 1)) *) + 0xce355273; (* arm_BCAX Q19 Q19 Q21 Q20 *) + 0x8af02409; (* arm_BIC X9 X0 (Shiftedreg X16 ROR 9) *) + 0xf9000ff8; (* arm_STR X24 SP (Immediate_Offset (word 24)) *) + 0xce360414; (* arm_BCAX Q20 Q0 Q22 Q1 *) + 0x8ae1b3b8; (* arm_BIC X24 X29 (Shiftedreg X1 ROR 44) *) + 0xce375835; (* arm_BCAX Q21 Q1 Q23 Q22 *) + 0x8af5c83b; (* arm_BIC X27 X1 (Shiftedreg X21 ROR 50) *) + 0x8afdff44; (* arm_BIC X4 X26 (Shiftedreg X29 ROR 63) *) + 0xce385ed6; (* arm_BCAX Q22 Q22 Q24 Q23 *) + 0xcac45421; (* arm_EOR X1 X1 (Shiftedreg X4 ROR 21) *) + 0xce2062f7; (* arm_BCAX Q23 Q23 Q0 Q24 *) + 0xf940016b; (* arm_LDR X11 X11 (Immediate_Offset (word 0)) *) + 0x8afee6a4; (* arm_BIC X4 X21 (Shiftedreg X30 ROR 57) *) + 0xce210318; (* arm_BCAX Q24 Q24 Q1 Q0 *) + 0xcad57b15; (* arm_EOR X21 X24 (Shiftedreg X21 ROR 30) *) + 0xcad3b138; (* arm_EOR X24 X9 (Shiftedreg X19 ROR 44) *) + 0xce226fc0; (* arm_BCAX Q0 Q30 Q2 Q27 *) + 0x8ae615c9; (* arm_BIC X9 X14 (Shiftedreg X6 ROR 5) *) + 0xce230b61; (* arm_BCAX Q1 Q27 Q3 Q2 *) + 0xcac0ad29; (* arm_EOR X9 X9 (Shiftedreg X0 ROR 43) *) + 0x8ae098c0; (* arm_BIC X0 X6 (Shiftedreg X0 ROR 38) *) + 0xce240c42; (* arm_BCAX Q2 Q2 Q4 Q3 *) + 0xca0b0021; (* arm_EOR X1 X1 X11 *) + 0xce3e1063; (* arm_BCAX Q3 Q3 Q30 Q4 *) + 0xcada8c8b; (* arm_EOR X11 X4 (Shiftedreg X26 ROR 35) *) + 0xcad0bc04; (* arm_EOR X4 X0 (Shiftedreg X16 ROR 47) *) + 0xce3b7884; (* arm_BCAX Q4 Q4 Q27 Q30 *) + 0x8af38e00; (* arm_BIC X0 X16 (Shiftedreg X19 ROR 35) *) + 0xcadeaf70; (* arm_EOR X16 X27 (Shiftedreg X30 ROR 43) *) + 0x8afaabdb; (* arm_BIC X27 X30 (Shiftedreg X26 ROR 42) *) + 0xf9400bfe; (* arm_LDR X30 SP (Immediate_Offset (word 16)) *) + 0x4ddfcfdc; (* arm_LD1R Q28 X30 (Postimmediate_Offset (word 8)) 64 128 *) + 0x8aeea67a; (* arm_BIC X26 X19 (Shiftedreg X14 ROR 41) *) + 0xf9000bfe; (* arm_STR X30 SP (Immediate_Offset (word 16)) *) + 0xcace3013; (* arm_EOR X19 X0 (Shiftedreg X14 ROR 12) *) + 0xcac6bb4e; (* arm_EOR X14 X26 (Shiftedreg X6 ROR 46) *) + 0x6e3c1c00; (* arm_EOR_VEC Q0 Q0 Q28 128 *) + 0xcadda766; (* arm_EOR X6 X27 (Shiftedreg X29 ROR 41) *) + 0xcacbd1e0; (* arm_EOR X0 X15 (Shiftedreg X11 ROR 52) *) + 0xcacdc000; (* arm_EOR X0 X0 (Shiftedreg X13 ROR 48) *) + 0xce05281e; (* arm_EOR3 Q30 Q0 Q5 Q10 *) + 0xce0f53de; (* arm_EOR3 Q30 Q30 Q15 Q20 *) + 0xcac9e51a; (* arm_EOR X26 X8 (Shiftedreg X9 ROR 57) *) + 0xcace281b; (* arm_EOR X27 X0 (Shiftedreg X14 ROR 10) *) + 0xce062c3d; (* arm_EOR3 Q29 Q1 Q6 Q11 *) + 0xcadcfe1d; (* arm_EOR X29 X16 (Shiftedreg X28 ROR 63) *) + 0xce1057bd; (* arm_EOR3 Q29 Q29 Q16 Q21 *) + 0xcac6cf5a; (* arm_EOR X26 X26 (Shiftedreg X6 ROR 51) *) + 0xcad6cafe; (* arm_EOR X30 X23 (Shiftedreg X22 ROR 50) *) + 0xce07305c; (* arm_EOR3 Q28 Q2 Q7 Q12 *) + 0xcaca7f40; (* arm_EOR X0 X26 (Shiftedreg X10 ROR 31) *) + 0xcad397bd; (* arm_EOR X29 X29 (Shiftedreg X19 ROR 37) *) + 0xce115b9c; (* arm_EOR3 Q28 Q28 Q17 Q22 *) + 0xcacc177b; (* arm_EOR X27 X27 (Shiftedreg X12 ROR 5) *) + 0xce08347b; (* arm_EOR3 Q27 Q3 Q8 Q13 *) + 0xcad88bde; (* arm_EOR X30 X30 (Shiftedreg X24 ROR 34) *) + 0xcac76c00; (* arm_EOR X0 X0 (Shiftedreg X7 ROR 27) *) + 0xce125f7b; (* arm_EOR3 Q27 Q27 Q18 Q23 *) + 0xcad56bda; (* arm_EOR X26 X30 (Shiftedreg X21 ROR 26) *) + 0xce09389a; (* arm_EOR3 Q26 Q4 Q9 Q14 *) + 0xcad93f5a; (* arm_EOR X26 X26 (Shiftedreg X25 ROR 15) *) + 0x93dbfb7e; (* arm_ROR X30 X27 62 *) + 0xce13635a; (* arm_EOR3 Q26 Q26 Q19 Q24 *) + 0xcadae7de; (* arm_EOR X30 X30 (Shiftedreg X26 ROR 57) *) + 0x93daeb5a; (* arm_ROR X26 X26 58 *) + 0xce7c8fd9; (* arm_RAX1 Q25 Q30 Q28 *) + 0xca1003d0; (* arm_EOR X16 X30 X16 *) + 0xce7a8f9c; (* arm_RAX1 Q28 Q28 Q26 *) + 0xcadcffdc; (* arm_EOR X28 X30 (Shiftedreg X28 ROR 63) *) + 0xf9006bfc; (* arm_STR X28 SP (Immediate_Offset (word 208)) *) + 0xce7d8f5a; (* arm_RAX1 Q26 Q26 Q29 *) + 0xcad193bd; (* arm_EOR X29 X29 (Shiftedreg X17 ROR 36) *) + 0xcac2f43c; (* arm_EOR X28 X1 (Shiftedreg X2 ROR 61) *) + 0xce7b8fbd; (* arm_RAX1 Q29 Q29 Q27 *) + 0xcad397d3; (* arm_EOR X19 X30 (Shiftedreg X19 ROR 37) *) + 0xce7e8f7b; (* arm_RAX1 Q27 Q27 Q30 *) + 0xcad40bbd; (* arm_EOR X29 X29 (Shiftedreg X20 ROR 2) *) + 0xcac4db9c; (* arm_EOR X28 X28 (Shiftedreg X4 ROR 54) *) + 0x6e3a1c1e; (* arm_EOR_VEC Q30 Q0 Q26 128 *) + 0xcac0df5a; (* arm_EOR X26 X26 (Shiftedreg X0 ROR 55) *) + 0xce9d0840; (* arm_XAR Q0 Q2 Q29 (word 2) *) + 0xcac39f9c; (* arm_EOR X28 X28 (Shiftedreg X3 ROR 39) *) + 0xcac5679c; (* arm_EOR X28 X28 (Shiftedreg X5 ROR 25) *) + 0xce9d5582; (* arm_XAR Q2 Q12 Q29 (word 21) *) + 0x93c0e000; (* arm_ROR X0 X0 56 *) + 0xcaddfc00; (* arm_EOR X0 X0 (Shiftedreg X29 ROR 63) *) + 0xce9c9dac; (* arm_XAR Q12 Q13 Q28 (word 39) *) + 0xcadbf79b; (* arm_EOR X27 X28 (Shiftedreg X27 ROR 61) *) + 0xce9be26d; (* arm_XAR Q13 Q19 Q27 (word 56) *) + 0xcacdb80d; (* arm_EOR X13 X0 (Shiftedreg X13 ROR 46) *) + 0xcadcffbc; (* arm_EOR X28 X29 (Shiftedreg X28 ROR 63) *) + 0xce9c22f3; (* arm_XAR Q19 Q23 Q28 (word 8) *) + 0xcad40bdd; (* arm_EOR X29 X30 (Shiftedreg X20 ROR 2) *) + 0xcac39f54; (* arm_EOR X20 X26 (Shiftedreg X3 ROR 39) *) + 0xce9a5df7; (* arm_XAR Q23 Q15 Q26 (word 23) *) + 0xcacbc80b; (* arm_EOR X11 X0 (Shiftedreg X11 ROR 50) *) + 0xce99fc2f; (* arm_XAR Q15 Q1 Q25 (word 63) *) + 0xcad92799; (* arm_EOR X25 X28 (Shiftedreg X25 ROR 9) *) + 0xcad55383; (* arm_EOR X3 X28 (Shiftedreg X21 ROR 20) *) + 0xce9c2501; (* arm_XAR Q1 Q8 Q28 (word 9) *) + 0xca010355; (* arm_EOR X21 X26 X1 *) + 0xce994e08; (* arm_XAR Q8 Q16 Q25 (word 19) *) + 0xcac9c769; (* arm_EOR X9 X27 (Shiftedreg X9 ROR 49) *) + 0xcad87398; (* arm_EOR X24 X28 (Shiftedreg X24 ROR 28) *) + 0xce9de8f0; (* arm_XAR Q16 Q7 Q29 (word 58) *) + 0xcad193c1; (* arm_EOR X1 X30 (Shiftedreg X17 ROR 36) *) + 0xcace200e; (* arm_EOR X14 X0 (Shiftedreg X14 ROR 8) *) + 0xce9af547; (* arm_XAR Q7 Q10 Q26 (word 61) *) + 0xcad6b396; (* arm_EOR X22 X28 (Shiftedreg X22 ROR 44) *) + 0xce9c906a; (* arm_XAR Q10 Q3 Q28 (word 36) *) + 0xcac8e368; (* arm_EOR X8 X27 (Shiftedreg X8 ROR 56) *) + 0xcac74f71; (* arm_EOR X17 X27 (Shiftedreg X7 ROR 19) *) + 0xce9cae43; (* arm_XAR Q3 Q18 Q28 (word 43) *) + 0xcacff80f; (* arm_EOR X15 X0 (Shiftedreg X15 ROR 62) *) + 0x8af6be87; (* arm_BIC X7 X20 (Shiftedreg X22 ROR 47) *) + 0xce9dc632; (* arm_XAR Q18 Q17 Q29 (word 49) *) + 0xcac4db44; (* arm_EOR X4 X26 (Shiftedreg X4 ROR 54) *) + 0xce99d971; (* arm_XAR Q17 Q11 Q25 (word 54) *) + 0xcacc0c00; (* arm_EOR X0 X0 (Shiftedreg X12 ROR 3) *) + 0xcad7eb9c; (* arm_EOR X28 X28 (Shiftedreg X23 ROR 58) *) + 0xce9bb12b; (* arm_XAR Q11 Q9 Q27 (word 44) *) + 0xcac2f757; (* arm_EOR X23 X26 (Shiftedreg X2 ROR 61) *) + 0xce9d0ec9; (* arm_XAR Q9 Q22 Q29 (word 3) *) + 0xcac5675a; (* arm_EOR X26 X26 (Shiftedreg X5 ROR 25) *) + 0xcad09ce2; (* arm_EOR X2 X7 (Shiftedreg X16 ROR 39) *) + 0xce9b65d6; (* arm_XAR Q22 Q14 Q27 (word 25) *) + 0x8af4a927; (* arm_BIC X7 X9 (Shiftedreg X20 ROR 42) *) + 0x8ae941fe; (* arm_BIC X30 X15 (Shiftedreg X9 ROR 16) *) + 0xce9aba8e; (* arm_XAR Q14 Q20 Q26 (word 46) *) + 0xcad664e7; (* arm_EOR X7 X7 (Shiftedreg X22 ROR 25) *) + 0xce9b9494; (* arm_XAR Q20 Q4 Q27 (word 37) *) + 0xcad4ebcc; (* arm_EOR X12 X30 (Shiftedreg X20 ROR 58) *) + 0x8af0e2d4; (* arm_BIC X20 X22 (Shiftedreg X16 ROR 56) *) + 0xce9bcb04; (* arm_XAR Q4 Q24 Q27 (word 50) *) + 0xcac6af7e; (* arm_EOR X30 X27 (Shiftedreg X6 ROR 43) *) + 0xcacf5e96; (* arm_EOR X22 X20 (Shiftedreg X15 ROR 23) *) + 0xce99fab8; (* arm_XAR Q24 Q21 Q25 (word 62) *) + 0x8aedaa66; (* arm_BIC X6 X19 (Shiftedreg X13 ROR 42) *) + 0xce9a70b5; (* arm_XAR Q21 Q5 Q26 (word 28) *) + 0xcad1a4c6; (* arm_EOR X6 X6 (Shiftedreg X17 ROR 41) *) + 0x8af1fda5; (* arm_BIC X5 X13 (Shiftedreg X17 ROR 63) *) + 0xce9950db; (* arm_XAR Q27 Q6 Q25 (word 20) *) + 0xcac556a5; (* arm_EOR X5 X21 (Shiftedreg X5 ROR 21) *) + 0xce272d45; (* arm_BCAX Q5 Q10 Q7 Q11 *) + 0x8af5b231; (* arm_BIC X17 X17 (Shiftedreg X21 ROR 44) *) + 0xcaca5f7b; (* arm_EOR X27 X27 (Shiftedreg X10 ROR 23) *) + 0xce281d66; (* arm_BCAX Q6 Q11 Q8 Q7 *) + 0x8af9cab5; (* arm_BIC X21 X21 (Shiftedreg X25 ROR 50) *) + 0x8ae46774; (* arm_BIC X20 X27 (Shiftedreg X4 ROR 25) *) + 0xce2920e7; (* arm_BCAX Q7 Q7 Q9 Q8 *) + 0x8aef7e0a; (* arm_BIC X10 X16 (Shiftedreg X15 ROR 31) *) + 0xce2a2508; (* arm_BCAX Q8 Q8 Q10 Q9 *) + 0xcad3aeb0; (* arm_EOR X16 X21 (Shiftedreg X19 ROR 43) *) + 0xcad97a35; (* arm_EOR X21 X17 (Shiftedreg X25 ROR 30) *) + 0xce2b2929; (* arm_BCAX Q9 Q9 Q11 Q10 *) + 0x8af3e733; (* arm_BIC X19 X25 (Shiftedreg X19 ROR 57) *) + 0xf9400ff9; (* arm_LDR X25 SP (Immediate_Offset (word 24)) *) + 0xce2c41ea; (* arm_BCAX Q10 Q15 Q12 Q16 *) + 0xcac9bd51; (* arm_EOR X17 X10 (Shiftedreg X9 ROR 47) *) + 0xce2d320b; (* arm_BCAX Q11 Q16 Q13 Q12 *) + 0xf94007e9; (* arm_LDR X9 SP (Immediate_Offset (word 8)) *) + 0xcadc6e8f; (* arm_EOR X15 X20 (Shiftedreg X28 ROR 27) *) + 0xce2e358c; (* arm_BCAX Q12 Q12 Q14 Q13 *) + 0x8afc0894; (* arm_BIC X20 X4 (Shiftedreg X28 ROR 2) *) + 0xce2f39ad; (* arm_BCAX Q13 Q13 Q15 Q14 *) + 0xcac1ca8a; (* arm_EOR X10 X20 (Shiftedreg X1 ROR 50) *) + 0x8afbf174; (* arm_BIC X20 X11 (Shiftedreg X27 ROR 60) *) + 0xce303dce; (* arm_BCAX Q14 Q14 Q16 Q15 *) + 0xcac45694; (* arm_EOR X20 X20 (Shiftedreg X4 ROR 21) *) + 0x8ae1c384; (* arm_BIC X4 X28 (Shiftedreg X1 ROR 48) *) + 0xce31568f; (* arm_BCAX Q15 Q20 Q17 Q21 *) + 0x8aebe421; (* arm_BIC X1 X1 (Shiftedreg X11 ROR 57) *) + 0xce3246b0; (* arm_BCAX Q16 Q21 Q18 Q17 *) + 0xf879793c; (* arm_LDR X28 X9 (Shiftreg_Offset X25 3) *) + 0xf9406be9; (* arm_LDR X9 SP (Immediate_Offset (word 208)) *) + 0xce334a31; (* arm_BCAX Q17 Q17 Q19 Q18 *) + 0x91000739; (* arm_ADD X25 X25 (rvalue (word 1)) *) + 0xf9000ff9; (* arm_STR X25 SP (Immediate_Offset (word 24)) *) + 0xce344e52; (* arm_BCAX Q18 Q18 Q20 Q19 *) + 0xf1005f3f; (* arm_CMP X25 (rvalue (word 23)) *) + 0xce355273; (* arm_BCAX Q19 Q19 Q21 Q20 *) + 0xcadbd439; (* arm_EOR X25 X1 (Shiftedreg X27 ROR 53) *) + 0x8afabfdb; (* arm_BIC X27 X30 (Shiftedreg X26 ROR 47) *) + 0xce360414; (* arm_BCAX Q20 Q0 Q22 Q1 *) + 0xca1c00a1; (* arm_EOR X1 X5 X28 *) + 0xce375835; (* arm_BCAX Q21 Q1 Q23 Q22 *) + 0xcacba485; (* arm_EOR X5 X4 (Shiftedreg X11 ROR 41) *) + 0xcacd8e6b; (* arm_EOR X11 X19 (Shiftedreg X13 ROR 35) *) + 0xce385ed6; (* arm_BCAX Q22 Q22 Q24 Q23 *) + 0x8af82b4d; (* arm_BIC X13 X26 (Shiftedreg X24 ROR 10) *) + 0xcad8e77c; (* arm_EOR X28 X27 (Shiftedreg X24 ROR 57) *) + 0xce2062f7; (* arm_BCAX Q23 Q23 Q0 Q24 *) + 0x8ae9bf1b; (* arm_BIC X27 X24 (Shiftedreg X9 ROR 47) *) + 0xce210318; (* arm_BCAX Q24 Q24 Q1 Q0 *) + 0x8ae326f3; (* arm_BIC X19 X23 (Shiftedreg X3 ROR 9) *) + 0x8aeea7a4; (* arm_BIC X4 X29 (Shiftedreg X14 ROR 41) *) + 0xce226fc0; (* arm_BCAX Q0 Q30 Q2 Q27 *) + 0xcaddb278; (* arm_EOR X24 X19 (Shiftedreg X29 ROR 44) *) + 0x8afd8c7d; (* arm_BIC X29 X3 (Shiftedreg X29 ROR 35) *) + 0xce230b61; (* arm_BCAX Q1 Q27 Q3 Q2 *) + 0xcac9e5ad; (* arm_EOR X13 X13 (Shiftedreg X9 ROR 57) *) + 0xce240c42; (* arm_BCAX Q2 Q2 Q4 Q3 *) + 0xcace33b3; (* arm_EOR X19 X29 (Shiftedreg X14 ROR 12) *) + 0x8ae04d3d; (* arm_BIC X29 X9 (Shiftedreg X0 ROR 19) *) + 0xce3e1063; (* arm_BCAX Q3 Q3 Q30 Q4 *) + 0x8ae815ce; (* arm_BIC X14 X14 (Shiftedreg X8 ROR 5) *) + 0xce3b7884; (* arm_BCAX Q4 Q4 Q27 Q30 *) + 0xcad7adc9; (* arm_EOR X9 X14 (Shiftedreg X23 ROR 43) *) + 0xcac8b88e; (* arm_EOR X14 X4 (Shiftedreg X8 ROR 46) *) + 0x8af79917; (* arm_BIC X23 X8 (Shiftedreg X23 ROR 38) *) + 0xcac00b68; (* arm_EOR X8 X27 (Shiftedreg X0 ROR 2) *) + 0xcac3bee4; (* arm_EOR X4 X23 (Shiftedreg X3 ROR 47) *) + 0x8afe1403; (* arm_BIC X3 X0 (Shiftedreg X30 ROR 5) *) + 0xcadad077; (* arm_EOR X23 X3 (Shiftedreg X26 ROR 52) *) + 0xcade63a3; (* arm_EOR X3 X29 (Shiftedreg X30 ROR 24) *) + 0xf9400bfe; (* arm_LDR X30 SP (Immediate_Offset (word 16)) *) + 0x4ddfcfdc; (* arm_LD1R Q28 X30 (Postimmediate_Offset (word 8)) 64 128 *) + 0xf9000bfe; (* arm_STR X30 SP (Immediate_Offset (word 16)) *) + 0x6e3c1c00; (* arm_EOR_VEC Q0 Q0 Q28 128 *) + 0x54ffe96d; (* arm_BLE (word 2096428) *) + 0x93c2f442; (* arm_ROR X2 X2 61 *) + 0x93c39c63; (* arm_ROR X3 X3 39 *) + 0x93c4d884; (* arm_ROR X4 X4 54 *) + 0x93c564a5; (* arm_ROR X5 X5 25 *) + 0x93c6acc6; (* arm_ROR X6 X6 43 *) + 0x93c74ce7; (* arm_ROR X7 X7 19 *) + 0x93c8e108; (* arm_ROR X8 X8 56 *) + 0x93c9c529; (* arm_ROR X9 X9 49 *) + 0x93ca5d4a; (* arm_ROR X10 X10 23 *) + 0x93cbc96b; (* arm_ROR X11 X11 50 *) + 0x93cc0d8c; (* arm_ROR X12 X12 3 *) + 0x93cdb9ad; (* arm_ROR X13 X13 46 *) + 0x93ce21ce; (* arm_ROR X14 X14 8 *) + 0x93cff9ef; (* arm_ROR X15 X15 62 *) + 0x93d19231; (* arm_ROR X17 X17 36 *) + 0x93dcff9c; (* arm_ROR X28 X28 63 *) + 0x93d39673; (* arm_ROR X19 X19 37 *) + 0x93d40a94; (* arm_ROR X20 X20 2 *) + 0x93d552b5; (* arm_ROR X21 X21 20 *) + 0x93d6b2d6; (* arm_ROR X22 X22 44 *) + 0x93d7eaf7; (* arm_ROR X23 X23 58 *) + 0x93d87318; (* arm_ROR X24 X24 28 *) + 0x93d92739; (* arm_ROR X25 X25 9 *) + 0xf94003e0; (* arm_LDR X0 SP (Immediate_Offset (word 0)) *) + 0x91064000; (* arm_ADD X0 X0 (rvalue (word 400)) *) + 0xa9001801; (* arm_STP X1 X6 X0 (Immediate_Offset (iword (&0))) *) + 0xa901400b; (* arm_STP X11 X16 X0 (Immediate_Offset (iword (&16))) *) + 0xa9020815; (* arm_STP X21 X2 X0 (Immediate_Offset (iword (&32))) *) + 0xa9033007; (* arm_STP X7 X12 X0 (Immediate_Offset (iword (&48))) *) + 0xa9045811; (* arm_STP X17 X22 X0 (Immediate_Offset (iword (&64))) *) + 0xa9052003; (* arm_STP X3 X8 X0 (Immediate_Offset (iword (&80))) *) + 0xa906700d; (* arm_STP X13 X28 X0 (Immediate_Offset (iword (&96))) *) + 0xa9071017; (* arm_STP X23 X4 X0 (Immediate_Offset (iword (&112))) *) + 0xa9083809; (* arm_STP X9 X14 X0 (Immediate_Offset (iword (&128))) *) + 0xa9096013; (* arm_STP X19 X24 X0 (Immediate_Offset (iword (&144))) *) + 0xa90a2805; (* arm_STP X5 X10 X0 (Immediate_Offset (iword (&160))) *) + 0xa90b500f; (* arm_STP X15 X20 X0 (Immediate_Offset (iword (&176))) *) + 0xf9006019; (* arm_STR X25 X0 (Immediate_Offset (word 192)) *) + 0xd1064000; (* arm_SUB X0 X0 (rvalue (word 400)) *) + 0x91032004; (* arm_ADD X4 X0 (rvalue (word 200)) *) + 0x4ec12819; (* arm_TRN1 Q25 Q0 Q1 64 128 *) + 0x4ec3285a; (* arm_TRN1 Q26 Q2 Q3 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec1681b; (* arm_TRN2 Q27 Q0 Q1 64 128 *) + 0x4ec3685c; (* arm_TRN2 Q28 Q2 Q3 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ec52899; (* arm_TRN1 Q25 Q4 Q5 64 128 *) + 0x4ec728da; (* arm_TRN1 Q26 Q6 Q7 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec5689b; (* arm_TRN2 Q27 Q4 Q5 64 128 *) + 0x4ec768dc; (* arm_TRN2 Q28 Q6 Q7 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ec92919; (* arm_TRN1 Q25 Q8 Q9 64 128 *) + 0x4ecb295a; (* arm_TRN1 Q26 Q10 Q11 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ec9691b; (* arm_TRN2 Q27 Q8 Q9 64 128 *) + 0x4ecb695c; (* arm_TRN2 Q28 Q10 Q11 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ecd2999; (* arm_TRN1 Q25 Q12 Q13 64 128 *) + 0x4ecf29da; (* arm_TRN1 Q26 Q14 Q15 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ecd699b; (* arm_TRN2 Q27 Q12 Q13 64 128 *) + 0x4ecf69dc; (* arm_TRN2 Q28 Q14 Q15 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ed12a19; (* arm_TRN1 Q25 Q16 Q17 64 128 *) + 0x4ed32a5a; (* arm_TRN1 Q26 Q18 Q19 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ed16a1b; (* arm_TRN2 Q27 Q16 Q17 64 128 *) + 0x4ed36a5c; (* arm_TRN2 Q28 Q18 Q19 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0x4ed52a99; (* arm_TRN1 Q25 Q20 Q21 64 128 *) + 0x4ed72ada; (* arm_TRN1 Q26 Q22 Q23 64 128 *) + 0xac816819; (* arm_STP Q25 Q26 X0 (Postimmediate_Offset (iword (&32))) *) + 0x4ed56a9b; (* arm_TRN2 Q27 Q20 Q21 64 128 *) + 0x4ed76adc; (* arm_TRN2 Q28 Q22 Q23 64 128 *) + 0x4c9fac9b; (* arm_STP Q27 Q28 X4 (Postimmediate_Offset (word 32)) *) + 0xfd000018; (* arm_STR D24 X0 (Immediate_Offset (word 0)) *) + 0x4ed86b19; (* arm_TRN2 Q25 Q24 Q24 64 128 *) + 0xfd000099; (* arm_STR D25 X4 (Immediate_Offset (word 0)) *) + 0x6d4927e8; (* arm_LDP D8 D9 SP (Immediate_Offset (iword (&144))) *) + 0x6d4a2fea; (* arm_LDP D10 D11 SP (Immediate_Offset (iword (&160))) *) + 0x6d4b37ec; (* arm_LDP D12 D13 SP (Immediate_Offset (iword (&176))) *) + 0x6d4c3fee; (* arm_LDP D14 D15 SP (Immediate_Offset (iword (&192))) *) + 0xa94353f3; (* arm_LDP X19 X20 SP (Immediate_Offset (iword (&48))) *) + 0xa9445bf5; (* arm_LDP X21 X22 SP (Immediate_Offset (iword (&64))) *) + 0xa94563f7; (* arm_LDP X23 X24 SP (Immediate_Offset (iword (&80))) *) + 0xa9466bf9; (* arm_LDP X25 X26 SP (Immediate_Offset (iword (&96))) *) + 0xa94773fb; (* arm_LDP X27 X28 SP (Immediate_Offset (iword (&112))) *) + 0xa9487bfd; (* arm_LDP X29 X30 SP (Immediate_Offset (iword (&128))) *) + 0x910383ff; (* arm_ADD SP SP (rvalue (word 224)) *) + 0xd65f03c0 (* arm_RET X30 *) +];; +(*** BYTECODE END ***) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC = ARM_MK_EXEC_RULE keccak_f1600_x3_v84a_scalar_mc;; + +(*** Additional lazy/deferred rotations in the implementation, row-major ***) + +let deferred_rotates = define + `deferred_rotates = + [ 0; 21; 14; 0; 44; + 3; 45; 61; 28; 20; + 25; 8; 18; 1; 6; + 10; 15; 56; 27; 36; + 39; 41; 2; 62; 55]`;; + +(* ------------------------------------------------------------------------- *) +(* Code length constants *) +(* ------------------------------------------------------------------------- *) + +let LENGTH_KECCAK_F1600_X3_V84A_SCALAR_HYBRID_MC = + REWRITE_CONV[keccak_f1600_x3_v84a_scalar_mc] `LENGTH keccak_f1600_x3_v84a_scalar_mc` + |> CONV_RULE (RAND_CONV LENGTH_CONV);; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH = 44`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH = 48`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START = KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH`;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END = new_definition + `KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END = LENGTH keccak_f1600_x3_v84a_scalar_mc - KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH`;; + +let LENGTH_SIMPLIFY_CONV = + REWRITE_CONV[LENGTH_KECCAK_F1600_X3_V84A_SCALAR_HYBRID_MC; + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START; KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END; + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_PREAMBLE_LENGTH; KECCAK_F1600_X3_V84A_SCALAR_HYBRID_POSTAMBLE_LENGTH] THENC + NUM_REDUCE_CONV THENC REWRITE_CONV [ADD_0];; + +(* ------------------------------------------------------------------------- *) +(* Correctness proof *) +(* ------------------------------------------------------------------------- *) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORRECT = prove + (`!a rc A1 A2 A3 pc stackpointer. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (stackpointer,216) /\ + ALLPAIRS nonoverlapping + [(a,600); (stackpointer,216)] + [(word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc); (rc,192)] + ==> ensures arm + (\s. aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word (pc + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_START) /\ + read SP s = stackpointer /\ + C_ARGUMENTS [a; rc] s /\ + wordlist_from_memory(a,25) s = A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = A3 /\ + wordlist_from_memory(rc,24) s = round_constants) + (\s. read PC s = word(pc + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORE_END) /\ + wordlist_from_memory(a,25) s = keccak 24 A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = + keccak 24 A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = + keccak 24 A3) + (MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI ,, + MAYCHANGE [X19; X20; X21; X22; X23; X24; X25; X26; + X27; X28; X29; X30] ,, + MAYCHANGE [Q8; Q9; Q10; Q11; Q12; Q13; Q14; Q15] ,, + MAYCHANGE [memory :> bytes(a,600); + memory :> bytes(stackpointer,32); + memory :> bytes(word_add stackpointer (word 208),8)])`, + CONV_TAC LENGTH_SIMPLIFY_CONV THEN + MAP_EVERY X_GEN_TAC + [`a:int64`; `rc:int64`; `A1:int64 list`; `A2:int64 list`; + `A3:int64 list`; `pc:num`; `stackpointer:int64`] THEN + REWRITE_TAC[fst KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC] THEN + REWRITE_TAC[MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI; C_ARGUMENTS; + ALL; ALLPAIRS; NONOVERLAPPING_CLAUSES] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN ASSUME_TAC) THEN + + (*** Establish once and for all that all the Ai have length 25 ***) + + ASM_CASES_TAC + `LENGTH(A1:int64 list) = 25 /\ LENGTH(A2:int64 list) = 25 /\ + LENGTH(A3:int64 list) = 25` + THENL + [FIRST_X_ASSUM(CONJUNCTS_THEN STRIP_ASSUME_TAC); + ENSURES_INIT_TAC "s0" THEN MATCH_MP_TAC(TAUT `F ==> p`) THEN + REPEAT(FIRST_X_ASSUM(MP_TAC o AP_TERM `LENGTH:int64 list->num`)) THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + REWRITE_TAC[LENGTH; ARITH] THEN ASM_MESON_TAC[]] THEN + + (*** The main loop: one scalar and one vector round per iteration ***) + + ENSURES_WHILE_PAUP_TAC `1` `24` `pc + 0x3dc` `pc + 0x6b0` + `\i s. + (read SP s = stackpointer /\ + wordlist_from_memory(rc,24) s:int64 list = round_constants /\ + read (memory :> bytes64 stackpointer) s = a /\ + read (memory :> bytes64 (word_add stackpointer (word 8))) s = rc /\ + read (memory :> bytes64 (word_add stackpointer (word 16))) s = + word_add rc (word(8 * i)) /\ + read (memory :> bytes64 (word_add stackpointer (word 24))) s = + word i /\ + [read Q0 s; read Q1 s; read Q2 s; read Q3 s; read Q4 s; + read Q5 s; read Q6 s; read Q7 s; read Q8 s; read Q9 s; + read Q10 s; read Q11 s; read Q12 s; read Q13 s; read Q14 s; + read Q15 s; read Q16 s; read Q17 s; read Q18 s; read Q19 s; + read Q20 s; read Q21 s; read Q22 s; read Q23 s; read Q24 s] = + MAP2 word_join (keccak i A2) (keccak i A1) /\ + [read X1 s; read X6 s; read X11 s; read X16 s; read X21 s; + read X2 s; read X7 s; read X12 s; read X17 s; read X22 s; + read X3 s; read X8 s; read X13 s; read X28 s; read X23 s; + read X4 s; read X9 s; read X14 s; read X19 s; read X24 s; + read X5 s; read X10 s; read X15 s; read X20 s; read X25 s] = + MAP2 word_rol (keccak i A3) (MAP (\j. 64 - j) deferred_rotates)) /\ + (condition_semantics Condition_LE s <=> i < 24)` THEN + REWRITE_TAC[condition_semantics] THEN REPEAT CONJ_TAC THENL + [ARITH_TAC; + + (*** Initial holding of the invariant ***) + + REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC; + WORDLIST_FROM_MEMORY_CONV `wordlist_from_memory(rc,24) s:int64 list`] THEN + ENSURES_INIT_TAC "s0" THEN + BIGNUM_DIGITIZE_TAC "A_" `read (memory :> bytes (a,8 * 75)) s0` THEN + REPEAT(FIRST_X_ASSUM + (MP_TAC o CONV_RULE(LAND_CONV WORDLIST_FROM_MEMORY_CONV))) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[] THEN REPEAT DISCH_TAC THEN + MEMORY_128_FROM_64_TAC "a" 0 12 THEN + MEMORY_128_FROM_64_TAC "a" 200 12 THEN + ASM_REWRITE_TAC[WORD_ADD_0] THEN REPEAT STRIP_TAC THEN + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--236) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + CONJ_TAC THENL [CONV_TAC WORD_RULE; ALL_TAC] THEN + CONJ_TAC THENL + [GEN_REWRITE_TAC (RAND_CONV o ONCE_DEPTH_CONV) + [ARITH_RULE `1 = 0 + 1`] THEN + REWRITE_TAC[keccak; round_constants; keccak_round] THEN + MAP_EVERY EXPAND_TAC ["A1"; "A2"; "A3"] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC; + + GEN_REWRITE_TAC (RAND_CONV o ONCE_DEPTH_CONV) + [ARITH_RULE `1 = 0 + 1`] THEN + REWRITE_TAC[keccak; round_constants; keccak_round; + deferred_rotates; MAP] THEN + CONV_TAC(RAND_CONV NUM_REDUCE_CONV) THEN + MAP_EVERY EXPAND_TAC ["A1"; "A2"; "A3"] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC]; + + (*** Preservation of the invariant including end condition code ***) + + X_GEN_TAC `i:num` THEN STRIP_TAC THEN + VAL_INT64_TAC `i:num` THEN + MP_TAC(WORD_RULE `word_add (word i) (word 1):int64 = word(i + 1)`) THEN + DISCH_TAC THEN + CONV_TAC(RATOR_CONV(LAND_CONV + (ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC] THEN + MP_TAC(ISPECL [`A1:int64 list`; `i:num`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A2:int64 list`; `i:num`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A3:int64 list`; `i:num`] LENGTH_KECCAK) THEN + ASM_REWRITE_TAC[IMP_IMP] THEN REWRITE_TAC[LENGTH_EQ_25] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN SUBST1_TAC) THEN + REWRITE_TAC[deferred_rotates; MAP] THEN CONV_TAC NUM_REDUCE_CONV THEN + REWRITE_TAC[MAP2; CONS_11; GSYM CONJ_ASSOC] THEN + ENSURES_INIT_TAC "s0" THEN + + (*** Both rc cursors read rc[i]: [sp+16] for the vector round and ***) + (*** [sp+8] indexed by the counter [sp+24] for the scalar round ***) + + SUBGOAL_THEN + `read (memory :> bytes64 (word_add rc (word(8 * i)))) s0 = + EL i round_constants` + ASSUME_TAC THENL + [UNDISCH_TAC `i < 24` THEN SPEC_TAC(`i:num`,`i:num`) THEN + CONV_TAC EXPAND_CASES_CONV THEN + CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV) THEN + ASM_REWRITE_TAC[round_constants; WORD_ADD_0] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN REWRITE_TAC[]; + ALL_TAC] THEN + + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--181) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + REWRITE_TAC[CONJ_ASSOC] THEN CONJ_TAC THENL + [REWRITE_TAC[GSYM CONJ_ASSOC]; + UNDISCH_TAC `i < 24` THEN SPEC_TAC(`i:num`,`i:num`) THEN + CONV_TAC EXPAND_CASES_CONV THEN + CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV)] THEN + CONJ_TAC THENL + [CONV_TAC(LAND_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[]; + ALL_TAC] THEN + CONJ_TAC THENL [CONV_TAC WORD_RULE; ALL_TAC] THEN + + (*** One round for both the vector and the scalar lanes ***) + + CONJ_TAC THEN + REWRITE_TAC[keccak; keccak_round] THEN + CONV_TAC(ONCE_DEPTH_CONV EL_CONV) THEN + CONV_TAC(TOP_DEPTH_CONV let_CONV) THEN + REWRITE_TAC[MAP2; CONS_11] THEN REPEAT CONJ_TAC THEN + KECCAK_BITBLAST_TAC; + + (*** The trivial loop-back goal ***) + + X_GEN_TAC `i:num` THEN STRIP_TAC THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ARM_SIM_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC [1] THEN + VAL_INT64_TAC `i:num` THEN + ASM_REWRITE_TAC[] THEN CONV_TAC(DEPTH_CONV WORD_NUM_RED_CONV); + + ALL_TAC] THEN + + (*** The finale making final deferred rotations and writing back ***) + + CONV_TAC NUM_REDUCE_CONV THEN + MP_TAC(ISPECL [`A1:int64 list`; `24`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A2:int64 list`; `24`] LENGTH_KECCAK) THEN + MP_TAC(ISPECL [`A3:int64 list`; `24`] LENGTH_KECCAK) THEN + ASM_REWRITE_TAC[] THEN + REWRITE_TAC[IMP_IMP] THEN REWRITE_TAC[LENGTH_EQ_25] THEN + DISCH_THEN(REPEAT_TCL CONJUNCTS_THEN SUBST1_TAC) THEN + REWRITE_TAC[deferred_rotates; MAP] THEN CONV_TAC NUM_REDUCE_CONV THEN + REWRITE_TAC[MAP2; CONS_11; GSYM CONJ_ASSOC] THEN + CONV_TAC(RATOR_CONV(LAND_CONV + (ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[round_constants; CONS_11; GSYM CONJ_ASSOC] THEN + REWRITE_TAC[GSYM round_constants] THEN + ENSURES_INIT_TAC "s0" THEN + ARM_STEPS_TAC KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC (1--80) THEN + ENSURES_FINAL_STATE_TAC THEN ASM_REWRITE_TAC[] THEN + CONV_TAC(ONCE_DEPTH_CONV WORDLIST_FROM_MEMORY_CONV) THEN + CONV_TAC(ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) THEN + ASM_REWRITE_TAC[] THEN + RULE_ASSUM_TAC(GEN_REWRITE_RULE TRY_CONV + [el 1 (CONJUNCTS READ_MEMORY_BYTESIZED_UNSPLIT)]) THEN + RULE_ASSUM_TAC(CONV_RULE + (ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV)) THEN + ASM_REWRITE_TAC[] THEN POP_ASSUM_LIST(K ALL_TAC) THEN + REWRITE_TAC[CONS_11] THEN REPEAT CONJ_TAC THEN CONV_TAC WORD_BLAST);; + +(* NOTE: This must be kept in sync with the CBMC specification + * in mldsa/src/fips202/native/aarch64/src/fips202_native_aarch64.h *) + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_CORRECT = prove + (`!a rc A1 A2 A3 pc stackpointer returnaddress. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (word_sub stackpointer (word 224),224) /\ + ALLPAIRS nonoverlapping + [(a,600); (word_sub stackpointer (word 224),224)] + [(word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc); (rc,192)] + ==> ensures arm + (\s. aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word pc /\ + read SP s = stackpointer /\ + read X30 s = returnaddress /\ + C_ARGUMENTS [a; rc] s /\ + wordlist_from_memory(a,25) s = A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = A3 /\ + wordlist_from_memory(rc,24) s = round_constants) + (\s. read PC s = returnaddress /\ + wordlist_from_memory(a,25) s = keccak 24 A1 /\ + wordlist_from_memory(word_add a (word 200),25) s = + keccak 24 A2 /\ + wordlist_from_memory(word_add a (word 400),25) s = + keccak 24 A3) + (MAYCHANGE_REGS_AND_FLAGS_PERMITTED_BY_ABI ,, + MAYCHANGE [memory :> bytes(a,600); + memory :> bytes(word_sub stackpointer (word 224),224)])`, + CONV_TAC LENGTH_SIMPLIFY_CONV THEN + let TWEAK_CONV = + ONCE_DEPTH_CONV + (WORDLIST_FROM_MEMORY_CONV THENC + ONCE_DEPTH_CONV NORMALIZE_RELATIVE_ADDRESS_CONV) in + CONV_TAC TWEAK_CONV THEN + ARM_ADD_RETURN_STACK_TAC ~pre_post_nsteps:(11,11) KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC + (CONV_RULE TWEAK_CONV (CONV_RULE LENGTH_SIMPLIFY_CONV KECCAK_F1600_X3_V84A_SCALAR_HYBRID_CORRECT)) + `[D8; D9; D10; D11; D12; D13; D14; D15; + X19; X20; X21; X22; X23; X24; X25; X26; X27; X28; X29; X30]` 224);; + +(* ------------------------------------------------------------------------- *) +(* Constant-time and memory safety proof. *) +(* ------------------------------------------------------------------------- *) + +needs "s2n_bignum/arm/proofs/consttime.ml";; +needs "mldsa_native/aarch64/proofs/subroutine_signatures.ml";; + +let full_spec,public_vars = mk_safety_spec + ~keep_maychanges:false + (assoc "sha3_keccak3_f1600_alt" subroutine_signatures) + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_CORRECT + KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC;; + +let KECCAK_F1600_X3_V84A_SCALAR_HYBRID_SUBROUTINE_SAFE = time prove + (`exists f_events. + forall e a rc pc stackpointer returnaddress. + aligned 16 stackpointer /\ + nonoverlapping (a,600) (word_sub stackpointer (word 224),224) /\ + ALLPAIRS nonoverlapping + [a,600; word_sub stackpointer (word 224),224] + [word pc,LENGTH keccak_f1600_x3_v84a_scalar_mc; rc,192] + ==> ensures arm + (\s. + aligned_bytes_loaded s (word pc) keccak_f1600_x3_v84a_scalar_mc /\ + read PC s = word pc /\ + read SP s = stackpointer /\ + read X30 s = returnaddress /\ + C_ARGUMENTS [a; rc] s /\ + read events s = e) + (\s. + read PC s = returnaddress /\ + exists e2. + read events s = APPEND e2 e /\ + e2 = + f_events rc a pc (word_sub stackpointer (word 224)) + returnaddress /\ + memaccess_inbounds e2 + [a,600; rc,192; + word_sub stackpointer (word 224),224] + [a,600; word_sub stackpointer (word 224),224]) + (\s s'. true)`, + ASSERT_CONCL_TAC full_spec THEN + PROVE_SAFETY_SPEC_TAC ~public_vars:public_vars KECCAK_F1600_X3_V84A_SCALAR_HYBRID_EXEC);; diff --git a/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml b/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml index 28c7f0b9d5..fe4d18b7e0 100644 --- a/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml +++ b/proofs/hol_light/aarch64/proofs/subroutine_signatures.ml @@ -373,4 +373,21 @@ let subroutine_signatures = [ [(* temporary buffers *) ]) ); + +("sha3_keccak3_f1600_alt", + ([(*args*) + ("a", "uint64_t[static 75]", (*is const?*)"false"); + ("rc", "uint64_t[static 24]", (*is const?*)"true"); + ], + "void", + [(* input buffers *) + ("a", "75"(* num elems *), 8(* elem bytesize *)); + ("rc", "24"(* num elems *), 8(* elem bytesize *)); + ], + [(* output buffers *) + ("a", "75"(* num elems *), 8(* elem bytesize *)); + ], + [(* temporary buffers *) + ]) +); ];; diff --git a/scripts/autogen b/scripts/autogen index 6ea7f7d48e..27bbd9f05c 100755 --- a/scripts/autogen +++ b/scripts/autogen @@ -3064,6 +3064,12 @@ def hol_light_asm_joblist(): f"-Imldsa/src/fips202/native/aarch64/src {aarch64_flags}", "aarch64", ), + ( + "keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S", + "dev/fips202/aarch64/src", + f"-Imldsa/src/fips202/native/aarch64/src {aarch64_flags}", + "aarch64", + ), ( "keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S", "dev/fips202/aarch64/src", diff --git a/test/abicheck/aarch64/abicheck_aarch64.mk b/test/abicheck/aarch64/abicheck_aarch64.mk index 0e5d2a3028..4403969845 100644 --- a/test/abicheck/aarch64/abicheck_aarch64.mk +++ b/test/abicheck/aarch64/abicheck_aarch64.mk @@ -21,6 +21,7 @@ ABICHECK_REQ_SHA3_FILES := ABICHECK_REQ_NEON_FILES := \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S \ + mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/native/aarch64/src/mldsa_intt_aarch64_asm.S \ @@ -47,6 +48,7 @@ $(ABICHECK_REQ_NEON_OBJS): CFLAGS += -march=armv8-a+simd ABICHECK_REQ_SHA3_FILES := \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S \ + mldsa/src/fips202/native/aarch64/src/keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.S \ mldsa/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S ABICHECK_REQ_SHA3_OBJS := $(call MAKE_OBJS,$(ABICHECK_DIR),$(ABICHECK_REQ_SHA3_FILES)) $(ABICHECK_REQ_SHA3_OBJS): CFLAGS += -march=armv8.4-a+sha3 diff --git a/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c b/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c new file mode 100644 index 0000000000..7bebe6aad9 --- /dev/null +++ b/test/abicheck/aarch64/checks/check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_aarch64.h" +#include "../checks_aarch64_all.h" + +#if defined(MLD_SYS_AARCH64) && defined(MLD_SYS_AARCH64_NEON) && \ + defined(__ARM_FEATURE_SHA3) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct aarch64_register_state reg_state; + +void mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(uint64_t state[75], + const uint64_t rc[24]); + +int check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_x0[600]; /* Three sequential Keccak states (state0[25], + state1[25], state2[25]) */ + MLD_ALIGN uint8_t buf_x1[192]; /* Round constants (24 x uint64_t) */ + + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON)) + { + fprintf(stderr, + "ABI check keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: host " + "lacks AArch64 NEON, skipping\n"); + return MLD_ABICHECK_SKIPPED; + } + + if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_SHA3)) + { + fprintf(stderr, + "ABI check keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm: host " + "lacks Armv8.4-A SHA3 (eor3, rax1, xar, bcax), skipping\n"); + return MLD_ABICHECK_SKIPPED; + } + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_aarch64_register_state(&input_state); + + randombytes(buf_x0, 600); + randombytes(buf_x1, 192); + + /* Set up register state for function arguments */ + input_state.gpr[0] = (uint64_t)buf_x0; + input_state.gpr[1] = (uint64_t)buf_x1; + + /* Call function through ABI test stub */ + call_stub_aarch64( + &input_state, &output_state, + (void (*)(void))mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm); + + /* Check ABI compliance */ + violations = check_aarch64_aapcs_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf( + stderr, + "ABI test FAILED for keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm " + "(iteration %d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_AARCH64 && MLD_SYS_AARCH64_NEON && __ARM_FEATURE_SHA3 */ diff --git a/test/abicheck/aarch64/checks_aarch64_all.h b/test/abicheck/aarch64/checks_aarch64_all.h index 3411639b80..d0ea90bccf 100644 --- a/test/abicheck/aarch64/checks_aarch64_all.h +++ b/test/abicheck/aarch64/checks_aarch64_all.h @@ -23,6 +23,7 @@ int check_keccak_f1600_x1_scalar_aarch64_asm(void); #if defined(__ARM_FEATURE_SHA3) int check_keccak_f1600_x1_v84a_aarch64_asm(void); int check_keccak_f1600_x2_v84a_aarch64_asm(void); +int check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm(void); #endif #endif /* MLD_SYS_AARCH64_NEON */ #if defined(MLD_SYS_AARCH64_NEON) @@ -62,6 +63,8 @@ static const abicheck_entry_t all_checks[] = { check_keccak_f1600_x1_v84a_aarch64_asm}, {"keccak_f1600_x2_v84a_aarch64_asm", check_keccak_f1600_x2_v84a_aarch64_asm}, + {"keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm", + check_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm}, #endif #endif /* MLD_SYS_AARCH64_NEON */ #if defined(MLD_SYS_AARCH64_NEON) diff --git a/test/mk/components.mk b/test/mk/components.mk index c3a51aadb9..93f85f5fad 100644 --- a/test/mk/components.mk +++ b/test/mk/components.mk @@ -260,6 +260,7 @@ ABICHECK_ASM_CFLAGS := \ -DMLD_FIPS202_AARCH64_NEED_X1_SCALAR \ -DMLD_FIPS202_AARCH64_NEED_X1_V84A \ -DMLD_FIPS202_AARCH64_NEED_X2_V84A \ + -DMLD_FIPS202_AARCH64_NEED_X3_V84A_SCALAR_HYBRID \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID \ -DMLD_FIPS202_ARMV81M_NEED_X4 \ From 8d7617765166e40ed36616cdf02ab4e9a4f72cb3 Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:14 +0000 Subject: [PATCH 4/8] Integration of the 3-way batched kernel with MLD_CONFIG_USE_X3 Sample three polynomials per XOF call when MLD_CONFIG_USE_X3 is set (Batch ExpandA, ExpandS and ExpandMask). Nonces are unchanged, so outputs match the default path. Signed-off-by: Soumya Bhattacharya --- mldsa/mldsa_native.c | 3 + mldsa/mldsa_native_asm.S | 3 + mldsa/src/fips202/fips202x4.c | 4 +- mldsa/src/fips202/fips202x4.h | 4 +- mldsa/src/fips202/keccakf1600.c | 2 + mldsa/src/fips202/keccakf1600.h | 2 + mldsa/src/poly.c | 65 +++++++++++++++ mldsa/src/poly.h | 34 ++++++++ mldsa/src/poly_kl.c | 143 +++++++++++++++++++++++++++++++- mldsa/src/poly_kl.h | 95 ++++++++++++++++++++- mldsa/src/polyvec.c | 45 +++++++++- mldsa/src/polyvec_lazy.c | 49 +++++++++++ mldsa/src/sign.c | 64 +++++++++++++- 13 files changed, 497 insertions(+), 16 deletions(-) diff --git a/mldsa/mldsa_native.c b/mldsa/mldsa_native.c index 00256a96ac..0d25663fe1 100644 --- a/mldsa/mldsa_native.c +++ b/mldsa/mldsa_native.c @@ -318,8 +318,10 @@ #undef mld_poly_challenge #undef mld_poly_decompose #undef mld_poly_uniform_eta +#undef mld_poly_uniform_eta_3x #undef mld_poly_uniform_eta_4x #undef mld_poly_uniform_gamma1 +#undef mld_poly_uniform_gamma1_3x #undef mld_poly_uniform_gamma1_4x #undef mld_poly_use_hint #undef mld_polyeta_pack @@ -487,6 +489,7 @@ #undef mld_poly_shiftl #undef mld_poly_sub #undef mld_poly_uniform +#undef mld_poly_uniform_3x #undef mld_poly_uniform_4x #undef mld_polyt0_pack #undef mld_polyt0_unpack diff --git a/mldsa/mldsa_native_asm.S b/mldsa/mldsa_native_asm.S index b1cfcba85d..0fd936bef4 100644 --- a/mldsa/mldsa_native_asm.S +++ b/mldsa/mldsa_native_asm.S @@ -356,8 +356,10 @@ #undef mld_poly_challenge #undef mld_poly_decompose #undef mld_poly_uniform_eta +#undef mld_poly_uniform_eta_3x #undef mld_poly_uniform_eta_4x #undef mld_poly_uniform_gamma1 +#undef mld_poly_uniform_gamma1_3x #undef mld_poly_uniform_gamma1_4x #undef mld_poly_use_hint #undef mld_polyeta_pack @@ -525,6 +527,7 @@ #undef mld_poly_shiftl #undef mld_poly_sub #undef mld_poly_uniform +#undef mld_poly_uniform_3x #undef mld_poly_uniform_4x #undef mld_polyt0_pack #undef mld_polyt0_unpack diff --git a/mldsa/src/fips202/fips202x4.c b/mldsa/src/fips202/fips202x4.c index 23da9ac800..c41a451730 100644 --- a/mldsa/src/fips202/fips202x4.c +++ b/mldsa/src/fips202/fips202x4.c @@ -15,7 +15,7 @@ #include "../common.h" #if !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ - !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) + !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && !defined(MLD_CONFIG_USE_X3) #include "../ct.h" #include "fips202.h" @@ -184,4 +184,4 @@ void mld_shake256x4_release(mld_shake256x4ctx *state) (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ #endif /* !MLD_CONFIG_MULTILEVEL_NO_SHARED && !MLD_CONFIG_SERIAL_FIPS202_ONLY \ - */ + && !MLD_CONFIG_USE_X3 */ diff --git a/mldsa/src/fips202/fips202x4.h b/mldsa/src/fips202/fips202x4.h index 2f5db18e6a..4814508770 100644 --- a/mldsa/src/fips202/fips202x4.h +++ b/mldsa/src/fips202/fips202x4.h @@ -8,7 +8,7 @@ #include "../common.h" -#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && !defined(MLD_CONFIG_USE_X3) #include @@ -121,5 +121,5 @@ void mld_shake256x4_release(mld_shake256x4ctx *state); #endif /* !MLD_CONFIG_NO_KEYPAIR_API || (!MLD_CONFIG_NO_SIGN_API && \ (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST)) */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ #endif /* !MLD_FIPS202_FIPS202X4_H */ diff --git a/mldsa/src/fips202/keccakf1600.c b/mldsa/src/fips202/keccakf1600.c index c8b0ad45bd..2cfeec8c9d 100644 --- a/mldsa/src/fips202/keccakf1600.c +++ b/mldsa/src/fips202/keccakf1600.c @@ -85,6 +85,7 @@ void mld_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, #if (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ defined(MLD_UNIT_TEST)) && \ !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) || defined(MLD_UNIT_TEST) static void mld_keccakf1600x4_extract_bytes_c(uint64_t *state, unsigned char *data0, unsigned char *data1, @@ -197,6 +198,7 @@ void mld_keccakf1600x4_permute(uint64_t *state) mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 2); mld_keccakf1600_permute(state + MLD_KECCAK_LANES * 3); } +#endif /* !MLD_CONFIG_USE_X3 || MLD_UNIT_TEST */ #if defined(MLD_CONFIG_USE_X3) /* 3-way (x3) batched Keccak primitives (see keccakf1600.h). diff --git a/mldsa/src/fips202/keccakf1600.h b/mldsa/src/fips202/keccakf1600.h index baae7e8e82..d9590e6359 100644 --- a/mldsa/src/fips202/keccakf1600.h +++ b/mldsa/src/fips202/keccakf1600.h @@ -51,6 +51,7 @@ __contract__( #if (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ defined(MLD_UNIT_TEST)) && \ !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) || defined(MLD_UNIT_TEST) #define mld_keccakf1600x4_extract_bytes \ MLD_NAMESPACE(keccakf1600x4_extract_bytes) MLD_INTERNAL_API @@ -101,6 +102,7 @@ __contract__( requires(memory_no_alias(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) assigns(memory_slice(state, sizeof(uint64_t) * MLD_KECCAK_LANES * MLD_KECCAK_WAY)) ); +#endif /* !MLD_CONFIG_USE_X3 || MLD_UNIT_TEST */ #if defined(MLD_CONFIG_USE_X3) /* 3-way (x3) batched Keccak primitives, used with MLD_CONFIG_USE_X3. They diff --git a/mldsa/src/poly.c b/mldsa/src/poly.c index 4153d80b8d..6974442652 100644 --- a/mldsa/src/poly.c +++ b/mldsa/src/poly.c @@ -685,6 +685,7 @@ void mld_poly_uniform(mld_poly *a, const uint8_t seed[MLDSA_SEEDBYTES + 2]) #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_4x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, mld_poly *vec3, @@ -749,6 +750,70 @@ void mld_poly_uniform_4x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, mld_zeroize(buf, sizeof(buf)); } +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandA path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE128 XOF, same 2-byte (x,y) + * nonce seed layout (set by the caller), same mld_rej_uniform mod-q rejection + * loop; only the batch width differs (3 lanes instead of 4). */ +MLD_INTERNAL_API +void mld_poly_uniform_3x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, + uint8_t seed[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLD_ALIGN uint8_t + buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_NBLOCKS * MLD_STREAM128_BLOCKBYTES)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[3]; + mld_xof128_x3_ctx state; + unsigned buflen; + + mld_xof128_x3_init(&state); + mld_xof128_x3_absorb(&state, seed, MLDSA_SEEDBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLD_POLY_UNIFORM_NBLOCKS. + * This should generate the matrix entries with high probability. + */ + + mld_xof128_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_NBLOCKS, &state); + buflen = MLD_POLY_UNIFORM_NBLOCKS * MLD_STREAM128_BLOCKBYTES; + ctr[0] = mld_rej_uniform(vec0->coeffs, MLDSA_N, 0, buf[0], buflen); + ctr[1] = mld_rej_uniform(vec1->coeffs, MLDSA_N, 0, buf[1], buflen); + ctr[2] = mld_rej_uniform(vec2->coeffs, MLDSA_N, 0, buf[2], buflen); + + /* + * So long as not all matrix entries have been generated, squeeze + * one more block a time until we're done. + */ + buflen = MLD_STREAM128_BLOCKBYTES; + while (ctr[0] < MLDSA_N || ctr[1] < MLDSA_N || ctr[2] < MLDSA_N) + __loop__( + assigns(ctr, state, object_whole(buf), + memory_slice(vec0, sizeof(mld_poly)), memory_slice(vec1, sizeof(mld_poly)), + memory_slice(vec2, sizeof(mld_poly))) + invariant(ctr[0] <= MLDSA_N && ctr[1] <= MLDSA_N && ctr[2] <= MLDSA_N) + invariant(array_bound(vec0->coeffs, 0, ctr[0], 0, MLDSA_Q)) + invariant(array_bound(vec1->coeffs, 0, ctr[1], 0, MLDSA_Q)) + invariant(array_bound(vec2->coeffs, 0, ctr[2], 0, MLDSA_Q))) + { + mld_xof128_x3_squeezeblocks(buf, 1, &state); + ctr[0] = mld_rej_uniform(vec0->coeffs, MLDSA_N, ctr[0], buf[0], buflen); + ctr[1] = mld_rej_uniform(vec1->coeffs, MLDSA_N, ctr[1], buf[1], buflen); + ctr[2] = mld_rej_uniform(vec2->coeffs, MLDSA_N, ctr[2], buf[2], buflen); + } + mld_xof128_x3_release(&state); + + mld_assert_bound(vec0->coeffs, MLDSA_N, 0, MLDSA_Q); + mld_assert_bound(vec1->coeffs, MLDSA_N, 0, MLDSA_Q); + mld_assert_bound(vec2->coeffs, MLDSA_N, 0, MLDSA_Q); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); +} +#endif /* MLD_CONFIG_USE_X3 */ + #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ diff --git a/mldsa/src/poly.h b/mldsa/src/poly.h index 91cc426b76..faa1e804cc 100644 --- a/mldsa/src/poly.h +++ b/mldsa/src/poly.h @@ -265,6 +265,7 @@ __contract__( #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_4x MLD_NAMESPACE(poly_uniform_4x) /** * Generate four polynomials using rejection sampling on (pseudo-)uniformly @@ -298,6 +299,39 @@ __contract__( ensures(array_bound(vec2->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) ensures(array_bound(vec3->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_3x MLD_NAMESPACE(poly_uniform_3x) +/** + * Generate three polynomials using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. 3-way variant of + * mld_poly_uniform_4x, used with MLD_CONFIG_USE_X3, used by the x3 ExpandA + * path. + * + * @spec{Implements @[FIPS204, Algorithm 30, RejNTTPoly] (three-way batched).} + * + * @param[out] vec0 Pointer to first polynomial to be sampled. + * @param[out] vec1 Pointer to second polynomial to be sampled. + * @param[out] vec2 Pointer to third polynomial to be sampled. + * @param[in] seed Pointer to consecutive array of seed buffers of size + * MLDSA_SEEDBYTES + 2 each, plus padding for alignment. + */ +MLD_INTERNAL_API +void mld_poly_uniform_3x(mld_poly *vec0, mld_poly *vec1, mld_poly *vec2, + uint8_t seed[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]) +__contract__( + requires(memory_no_alias(vec0, sizeof(mld_poly))) + requires(memory_no_alias(vec1, sizeof(mld_poly))) + requires(memory_no_alias(vec2, sizeof(mld_poly))) + requires(memory_no_alias(seed, 3 * MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2))) + assigns(memory_slice(vec0, sizeof(mld_poly))) + assigns(memory_slice(vec1, sizeof(mld_poly))) + assigns(memory_slice(vec2, sizeof(mld_poly))) + ensures(array_bound(vec0->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) + ensures(array_bound(vec1->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) + ensures(array_bound(vec2->coeffs, 0, MLDSA_N, 0, MLDSA_Q)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ diff --git a/mldsa/src/poly_kl.c b/mldsa/src/poly_kl.c index 95241c08a5..ab9af0754b 100644 --- a/mldsa/src/poly_kl.c +++ b/mldsa/src/poly_kl.c @@ -322,6 +322,7 @@ __contract__( } #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_eta_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_poly *r3, const uint8_t seed[MLDSA_CRHBYTES], @@ -405,6 +406,85 @@ void mld_poly_uniform_eta_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } + +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_eta_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandS path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE256 XOF, same 1-byte nonce + * seed layout, same mld_rej_eta rejection loop; only the batch width differs + * (3 lanes instead of 4). */ +MLD_INTERNAL_API +void mld_poly_uniform_eta_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], uint8_t nonce0, + uint8_t nonce1, uint8_t nonce2) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLD_ALIGN uint8_t buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_ETA_NBLOCKS * + MLD_STREAM256_BLOCKBYTES)]; + + MLD_ALIGN uint8_t extseed[3][MLD_ALIGN_UP(MLDSA_CRHBYTES + 2)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[3]; + mld_xof256_x3_ctx state; + unsigned buflen; + + mld_memcpy(extseed[0], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[1], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[2], seed, MLDSA_CRHBYTES); + extseed[0][MLDSA_CRHBYTES] = nonce0; + extseed[1][MLDSA_CRHBYTES] = nonce1; + extseed[2][MLDSA_CRHBYTES] = nonce2; + extseed[0][MLDSA_CRHBYTES + 1] = 0; + extseed[1][MLDSA_CRHBYTES + 1] = 0; + extseed[2][MLDSA_CRHBYTES + 1] = 0; + + mld_xof256_x3_init(&state); + mld_xof256_x3_absorb(&state, extseed, MLDSA_CRHBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLD_POLY_UNIFORM_ETA_NBLOCKS. + * This should generate the coefficients with high probability. + */ + mld_xof256_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_ETA_NBLOCKS, &state); + buflen = MLD_POLY_UNIFORM_ETA_NBLOCKS * MLD_STREAM256_BLOCKBYTES; + + ctr[0] = mld_rej_eta(r0->coeffs, MLDSA_N, 0, buf[0], buflen); + ctr[1] = mld_rej_eta(r1->coeffs, MLDSA_N, 0, buf[1], buflen); + ctr[2] = mld_rej_eta(r2->coeffs, MLDSA_N, 0, buf[2], buflen); + + /* + * So long as not all entries have been generated, squeeze + * one more block at a time until we're done. + */ + buflen = MLD_STREAM256_BLOCKBYTES; + while (ctr[0] < MLDSA_N || ctr[1] < MLDSA_N || ctr[2] < MLDSA_N) + __loop__( + assigns(ctr, state, memory_slice(r0, sizeof(mld_poly)), + memory_slice(r1, sizeof(mld_poly)), memory_slice(r2, sizeof(mld_poly)), + object_whole(buf[0]), object_whole(buf[1]), object_whole(buf[2])) + invariant(ctr[0] <= MLDSA_N && ctr[1] <= MLDSA_N && ctr[2] <= MLDSA_N) + invariant(array_abs_bound(r0->coeffs, 0, ctr[0], MLDSA_ETA + 1)) + invariant(array_abs_bound(r1->coeffs, 0, ctr[1], MLDSA_ETA + 1)) + invariant(array_abs_bound(r2->coeffs, 0, ctr[2], MLDSA_ETA + 1))) + { + mld_xof256_x3_squeezeblocks(buf, 1, &state); + ctr[0] = mld_rej_eta(r0->coeffs, MLDSA_N, ctr[0], buf[0], buflen); + ctr[1] = mld_rej_eta(r1->coeffs, MLDSA_N, ctr[1], buf[1], buflen); + ctr[2] = mld_rej_eta(r2->coeffs, MLDSA_N, ctr[2], buf[2], buflen); + } + + mld_xof256_x3_release(&state); + + mld_assert_abs_bound(r0->coeffs, MLDSA_N, MLDSA_ETA + 1); + mld_assert_abs_bound(r1->coeffs, MLDSA_N, MLDSA_ETA + 1); + mld_assert_abs_bound(r2->coeffs, MLDSA_N, MLDSA_ETA + 1); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); + mld_zeroize(extseed, sizeof(extseed)); +} +#endif /* MLD_CONFIG_USE_X3 */ #else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ MLD_INTERNAL_API @@ -470,8 +550,12 @@ void mld_poly_uniform_eta(mld_poly *r, const uint8_t seed[MLDSA_CRHBYTES], ((MLDSA_POLYZ_PACKEDBYTES + MLD_STREAM256_BLOCKBYTES - 1) / \ MLD_STREAM256_BLOCKBYTES) -#if MLD_CONFIG_PARAMETER_SET == 65 || \ - defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ +/* The serial sampler is used by the default x4 ExpandMask for L=5, and by the + * x3 ExpandMask (MLD_CONFIG_USE_X3) for its 1-polynomial remainder (L=4 and + * L=7). */ +#if (MLD_CONFIG_PARAMETER_SET == 65 && !defined(MLD_CONFIG_USE_X3)) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && defined(MLD_CONFIG_USE_X3)) || \ + defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) MLD_INTERNAL_API void mld_poly_uniform_gamma1(mld_poly *a, const uint8_t seed[MLDSA_CRHBYTES], @@ -500,12 +584,15 @@ void mld_poly_uniform_gamma1(mld_poly *a, const uint8_t seed[MLDSA_CRHBYTES], mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } -#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_SERIAL_FIPS202_ONLY || \ - MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ +#endif /* (MLD_CONFIG_PARAMETER_SET == 65 && !MLD_CONFIG_USE_X3) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && MLD_CONFIG_USE_X3) || \ + MLD_CONFIG_SERIAL_FIPS202_ONLY || MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) MLD_INTERNAL_API void mld_poly_uniform_gamma1_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_poly *r3, @@ -554,6 +641,54 @@ void mld_poly_uniform_gamma1_4x(mld_poly *r0, mld_poly *r1, mld_poly *r2, mld_zeroize(buf, sizeof(buf)); mld_zeroize(extseed, sizeof(extseed)); } + +#else /* !MLD_CONFIG_USE_X3 */ +/* 3-way variant of mld_poly_uniform_gamma1_4x (used with + * MLD_CONFIG_USE_X3), used by the x3 ExpandMask path. Byte-for-byte identical + * per-lane behaviour to the x4 sampler: same SHAKE256 XOF, same 2-byte nonce + * layout, same mld_polyz_unpack; only the batch width differs (3 lanes). */ +MLD_INTERNAL_API +void mld_poly_uniform_gamma1_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], + uint16_t nonce0, uint16_t nonce1, + uint16_t nonce2) +{ + /* Temporary buffers for XOF output before unpacking */ + MLD_ALIGN uint8_t buf[3][MLD_ALIGN_UP(MLD_POLY_UNIFORM_GAMMA1_NBLOCKS * + MLD_STREAM256_BLOCKBYTES)]; + + MLD_ALIGN uint8_t extseed[3][MLD_ALIGN_UP(MLDSA_CRHBYTES + 2)]; + + mld_xof256_x3_ctx state; + + mld_memcpy(extseed[0], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[1], seed, MLDSA_CRHBYTES); + mld_memcpy(extseed[2], seed, MLDSA_CRHBYTES); + extseed[0][MLDSA_CRHBYTES] = (uint8_t)(nonce0 & 0xFF); + extseed[1][MLDSA_CRHBYTES] = (uint8_t)(nonce1 & 0xFF); + extseed[2][MLDSA_CRHBYTES] = (uint8_t)(nonce2 & 0xFF); + extseed[0][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce0 >> 8); + extseed[1][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce1 >> 8); + extseed[2][MLDSA_CRHBYTES + 1] = (uint8_t)(nonce2 >> 8); + + mld_xof256_x3_init(&state); + mld_xof256_x3_absorb(&state, extseed, MLDSA_CRHBYTES + 2); + mld_xof256_x3_squeezeblocks(buf, MLD_POLY_UNIFORM_GAMMA1_NBLOCKS, &state); + + mld_polyz_unpack(r0, buf[0]); + mld_polyz_unpack(r1, buf[1]); + mld_polyz_unpack(r2, buf[2]); + mld_xof256_x3_release(&state); + + mld_assert_bound(r0->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + mld_assert_bound(r1->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + mld_assert_bound(r2->coeffs, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); + + /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ + mld_zeroize(buf, sizeof(buf)); + mld_zeroize(extseed, sizeof(extseed)); +} +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ #endif /* !MLD_CONFIG_NO_SIGN_API */ diff --git a/mldsa/src/poly_kl.h b/mldsa/src/poly_kl.h index d31510ceb9..dfad54e7f2 100644 --- a/mldsa/src/poly_kl.h +++ b/mldsa/src/poly_kl.h @@ -72,6 +72,7 @@ __contract__( #if !defined(MLD_CONFIG_NO_KEYPAIR_API) #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_eta_4x MLD_NAMESPACE_KL(poly_uniform_eta_4x) /** * Sample four polynomials with uniformly random coefficients in @@ -111,6 +112,44 @@ __contract__( ensures(array_abs_bound(r2->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) ensures(array_abs_bound(r3->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_eta_3x MLD_NAMESPACE_KL(poly_uniform_eta_3x) +/** + * Sample three polynomials with uniformly random coefficients in + * [-MLDSA_ETA, MLDSA_ETA] by performing rejection sampling on the output + * stream from SHAKE256(seed|nonce_i). 3-way variant of + * mld_poly_uniform_eta_4x, used with MLD_CONFIG_USE_X3, used by the x3 + * ExpandS path. + * + * @spec{Implements @[FIPS204, Algorithm 31, RejBoundedPoly] (three-way + * batched).} + * + * @param[out] r0 Pointer to first output polynomial. + * @param[out] r1 Pointer to second output polynomial. + * @param[out] r2 Pointer to third output polynomial. + * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. + * @param nonce0 First nonce. + * @param nonce1 Second nonce. + * @param nonce2 Third nonce. + */ +MLD_INTERNAL_API +void mld_poly_uniform_eta_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], uint8_t nonce0, + uint8_t nonce1, uint8_t nonce2) +__contract__( + requires(memory_no_alias(r0, sizeof(mld_poly))) + requires(memory_no_alias(r1, sizeof(mld_poly))) + requires(memory_no_alias(r2, sizeof(mld_poly))) + requires(memory_no_alias(seed, MLDSA_CRHBYTES)) + assigns(memory_slice(r0, sizeof(mld_poly))) + assigns(memory_slice(r1, sizeof(mld_poly))) + assigns(memory_slice(r2, sizeof(mld_poly))) + ensures(array_abs_bound(r0->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) + ensures(array_abs_bound(r1->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) + ensures(array_abs_bound(r2->coeffs, 0, MLDSA_N, MLDSA_ETA + 1)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ #if defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) @@ -139,8 +178,12 @@ __contract__( #endif /* !MLD_CONFIG_NO_KEYPAIR_API */ #if !defined(MLD_CONFIG_NO_SIGN_API) -#if MLD_CONFIG_PARAMETER_SET == 65 || \ - defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ +/* The serial sampler is used by the default x4 ExpandMask for L=5, and by the + * x3 ExpandMask (MLD_CONFIG_USE_X3) for its 1-polynomial remainder (L=4 and + * L=7). */ +#if (MLD_CONFIG_PARAMETER_SET == 65 && !defined(MLD_CONFIG_USE_X3)) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && defined(MLD_CONFIG_USE_X3)) || \ + defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) || \ defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) #define mld_poly_uniform_gamma1 MLD_NAMESPACE_KL(poly_uniform_gamma1) /** @@ -164,11 +207,14 @@ __contract__( assigns(memory_slice(a, sizeof(mld_poly))) ensures(array_bound(a->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ); -#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_SERIAL_FIPS202_ONLY || \ - MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ +#endif /* (MLD_CONFIG_PARAMETER_SET == 65 && !MLD_CONFIG_USE_X3) || \ + (MLD_CONFIG_PARAMETER_SET != 65 && MLD_CONFIG_USE_X3) || \ + MLD_CONFIG_SERIAL_FIPS202_ONLY || MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST */ #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) +#if !defined(MLD_CONFIG_USE_X3) #define mld_poly_uniform_gamma1_4x MLD_NAMESPACE_KL(poly_uniform_gamma1_4x) /** * Sample four polynomials with uniformly random coefficients in @@ -209,6 +255,47 @@ __contract__( ensures(array_bound(r2->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ensures(array_bound(r3->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ); + +#else /* !MLD_CONFIG_USE_X3 */ +#define mld_poly_uniform_gamma1_3x MLD_NAMESPACE_KL(poly_uniform_gamma1_3x) +/** + * Sample three polynomials with uniformly random coefficients in + * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1] by unpacking output streams of + * SHAKE256(seed|nonce_i). 3-way variant of + * mld_poly_uniform_gamma1_4x, used with MLD_CONFIG_USE_X3, used by the x3 + * ExpandMask path. Byte-for-byte identical per-lane behaviour to the x4 + * sampler: same SHAKE256 XOF, same 2-byte nonce layout, same mld_polyz_unpack; + * only the batch width differs (3 lanes instead of 4). + * + * @spec{Partially implements @[FIPS204, Algorithm 34, ExpandMask] (three-way + * batched).} + * + * @param[out] r0 Pointer to first output polynomial. + * @param[out] r1 Pointer to second output polynomial. + * @param[out] r2 Pointer to third output polynomial. + * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. + * @param nonce0 First 16-bit nonce. + * @param nonce1 Second 16-bit nonce. + * @param nonce2 Third 16-bit nonce. + */ +MLD_INTERNAL_API +void mld_poly_uniform_gamma1_3x(mld_poly *r0, mld_poly *r1, mld_poly *r2, + const uint8_t seed[MLDSA_CRHBYTES], + uint16_t nonce0, uint16_t nonce1, + uint16_t nonce2) +__contract__( + requires(memory_no_alias(r0, sizeof(mld_poly))) + requires(memory_no_alias(r1, sizeof(mld_poly))) + requires(memory_no_alias(r2, sizeof(mld_poly))) + requires(memory_no_alias(seed, MLDSA_CRHBYTES)) + assigns(memory_slice(r0, sizeof(mld_poly))) + assigns(memory_slice(r1, sizeof(mld_poly))) + assigns(memory_slice(r2, sizeof(mld_poly))) + ensures(array_bound(r0->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) + ensures(array_bound(r1->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) + ensures(array_bound(r2->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) +); +#endif /* MLD_CONFIG_USE_X3 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ MLD_UNIT_TEST) */ #endif /* !MLD_CONFIG_NO_SIGN_API */ diff --git a/mldsa/src/polyvec.c b/mldsa/src/polyvec.c index be13ae3f91..69a8ba33df 100644 --- a/mldsa/src/polyvec.c +++ b/mldsa/src/polyvec.c @@ -17,6 +17,10 @@ #include "debug.h" #include "polyvec_lazy.h" +#if defined(MLD_CONFIG_USE_X3) +#include "ct.h" /* mld_zeroize, for the x3 ExpandMask scratch-lane remainder */ +#endif + /* This namespacing is not done at the top to avoid a naming conflict * with native backends, which are currently not yet namespaced. */ #define mld_polyvecl_pointwise_acc_montgomery_c \ @@ -44,7 +48,44 @@ void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, { mld_poly_uniform_gamma1(&v->vec[i], seed, (uint16_t)(kappa + i)); } -#else /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#elif defined(MLD_CONFIG_USE_X3) + /* + * x3 ExpandMask (MLD_CONFIG_USE_X3): sample the L + * mask polynomials 3 at a time via mld_poly_uniform_gamma1_3x. The nonce + * assigned to each polynomial is IDENTICAL to the x4 path below (poly i uses + * nonce kappa + i), so the sampled vector is byte-for-byte the same; only the + * batch width (3 vs 4) differs. Any sub-3 remainder is sampled serially + * (L=4, L=7) or via a scratch 3rd lane with a throwaway nonce (L=5). + */ +#if MLDSA_L == 4 + /* L = 4 = 1 + 3: one serial poly + one 3-way batch. */ + mld_poly_uniform_gamma1(&v->vec[0], seed, kappa); + mld_poly_uniform_gamma1_3x(&v->vec[1], &v->vec[2], &v->vec[3], seed, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2), + (uint16_t)(kappa + 3)); +#elif MLDSA_L == 5 + /* L = 5 = 3 + 2: two 3-way batches, the 3rd lane of the 2nd batch is a + * throwaway (scratch poly, nonce 0xFF, zeroized after). */ + { + MLD_ALIGN mld_poly scratch; + mld_poly_uniform_gamma1_3x(&v->vec[0], &v->vec[1], &v->vec[2], seed, kappa, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2)); + mld_poly_uniform_gamma1_3x(&v->vec[3], &v->vec[4], &scratch, seed, + (uint16_t)(kappa + 3), (uint16_t)(kappa + 4), + 0xFF /* irrelevant */); + mld_zeroize(&scratch, sizeof(scratch)); + } +#elif MLDSA_L == 7 + /* L = 7 = 1 + 3 + 3: one serial poly + two 3-way batches. */ + mld_poly_uniform_gamma1(&v->vec[0], seed, kappa); + mld_poly_uniform_gamma1_3x(&v->vec[1], &v->vec[2], &v->vec[3], seed, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2), + (uint16_t)(kappa + 3)); + mld_poly_uniform_gamma1_3x(&v->vec[4], &v->vec[5], &v->vec[6], seed, + (uint16_t)(kappa + 4), (uint16_t)(kappa + 5), + (uint16_t)(kappa + 6)); +#endif /* MLDSA_L == 7 */ +#else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ #if MLDSA_L == 4 mld_poly_uniform_gamma1_4x(&v->vec[0], &v->vec[1], &v->vec[2], &v->vec[3], seed, kappa, (uint16_t)(kappa + 1), @@ -63,7 +104,7 @@ void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, seed, (uint16_t)(kappa + 3), (uint16_t)(kappa + 4), (uint16_t)(kappa + 5), (uint16_t)(kappa + 6)); #endif /* MLDSA_L == 7 */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ mld_assert_bound_2d(v->vec, MLDSA_L, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1); diff --git a/mldsa/src/polyvec_lazy.c b/mldsa/src/polyvec_lazy.c index dab3ce96a9..3e06ba2676 100644 --- a/mldsa/src/polyvec_lazy.c +++ b/mldsa/src/polyvec_lazy.c @@ -76,6 +76,54 @@ void mld_polyvec_matrix_expand_eager(mld_polymat_eager *mat, } #if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) +#if defined(MLD_CONFIG_USE_X3) + /* + * x3 ExpandA (MLD_CONFIG_USE_X3): sample 3 matrix + * entries at a time via mld_poly_uniform_3x. The (x, y) nonce assignment and + * sampling order are identical to the x4 path below, so the sampled matrix A + * is byte-for-byte the same; only the batch width (3 vs 4) differs. Any + * remainder (K*L % 3) is sampled individually by the shared serial tail loop + * below. + */ + for (i = 0; i < (MLDSA_K * MLDSA_L / 3) * 3; i += 3) + __loop__( + assigns(i, j, object_whole(seed_ext), memory_slice(mat, sizeof(mld_polymat_eager))) + invariant(i <= (MLDSA_K * MLDSA_L / 3) * 3 && i % 3 == 0) + /* vectors 0 .. i / MLDSA_L are completely sampled */ + invariant(forall(k1, 0, i / MLDSA_L, forall(l1, 0, MLDSA_L, + array_bound(mat->vec[k1].vec[l1].coeffs, 0, MLDSA_N, 0, MLDSA_Q)))) + /* last vector is sampled up to i % MLDSA_L */ + invariant(forall(k2, i / MLDSA_L, i / MLDSA_L + 1, forall(l2, 0, i % MLDSA_L, + array_bound(mat->vec[k2].vec[l2].coeffs, 0, MLDSA_N, 0, MLDSA_Q)))) + decreases((MLDSA_K * MLDSA_L / 3) * 3 - i) + ) + { + for (j = 0; j < 3; j++) + __loop__( + assigns(j, object_whole(seed_ext)) + invariant(j <= 3) + decreases(3 - j) + ) + { + uint8_t x = (uint8_t)((i + j) / MLDSA_L); + uint8_t y = (uint8_t)((i + j) % MLDSA_L); + + seed_ext[j][MLDSA_SEEDBYTES + 0] = y; + seed_ext[j][MLDSA_SEEDBYTES + 1] = x; + } + + mld_poly_uniform_3x(&mat->vec[i / MLDSA_L].vec[i % MLDSA_L], + &mat->vec[(i + 1) / MLDSA_L].vec[(i + 1) % MLDSA_L], + &mat->vec[(i + 2) / MLDSA_L].vec[(i + 2) % MLDSA_L], + seed_ext); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[i / MLDSA_L].vec[i % MLDSA_L]); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[(i + 1) / MLDSA_L].vec[(i + 1) % MLDSA_L]); + mld_poly_permute_bitrev_to_custom_optional( + &mat->vec[(i + 2) / MLDSA_L].vec[(i + 2) % MLDSA_L]); + } +#else /* MLD_CONFIG_USE_X3 */ /* Sample 4 matrix entries a time. */ for (i = 0; i < (MLDSA_K * MLDSA_L / 4) * 4; i += 4) __loop__( @@ -118,6 +166,7 @@ void mld_polyvec_matrix_expand_eager(mld_polymat_eager *mat, mld_poly_permute_bitrev_to_custom_optional( &mat->vec[(i + 3) / MLDSA_L].vec[(i + 3) % MLDSA_L]); } +#endif /* !MLD_CONFIG_USE_X3 */ #else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ i = 0; #endif /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/mldsa/src/sign.c b/mldsa/src/sign.c index 48dac1184b..c597a0aa07 100644 --- a/mldsa/src/sign.c +++ b/mldsa/src/sign.c @@ -227,7 +227,67 @@ __contract__( mld_poly_uniform_eta(&s1->vec[i], seed, (uint8_t)i); mld_polyeta_pack(s1_packed + i * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[i]); } -#else /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#elif defined(MLD_CONFIG_USE_X3) +/* x3 ExpandS (MLD_CONFIG_USE_X3): same nonce assignment and the same + * s2-first, s1-as-scratch order as the 4-way path below, with batches of 3. + * A lane that is not needed carries nonce 0xFF and writes an s1 slot that is + * sampled afterwards. */ +#if MLD_CONFIG_PARAMETER_SET == 44 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 4, 5, 6); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[3], &s1->vec[1], seed, 7, 3, + 0xFF /* irrelevant */); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#elif MLD_CONFIG_PARAMETER_SET == 65 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 5, 6, 7); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 8, 9, + 10); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[3], &s1->vec[4], &s1->vec[0], seed, 3, 4, + 0xFF /* irrelevant */); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_polyeta_pack(s1_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[4]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#elif MLD_CONFIG_PARAMETER_SET == 87 + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 7, 8, 9); + mld_polyeta_pack(s2_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 10, 11, + 12); + mld_polyeta_pack(s2_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s2_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[6], seed, 13, 14, + 6); + mld_polyeta_pack(s2_packed + 6 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s2_packed + 7 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 6 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[6]); + mld_poly_uniform_eta_3x(&s1->vec[3], &s1->vec[4], &s1->vec[5], seed, 3, 4, 5); + mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); + mld_polyeta_pack(s1_packed + 4 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[4]); + mld_polyeta_pack(s1_packed + 5 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[5]); + mld_poly_uniform_eta_3x(&s1->vec[0], &s1->vec[1], &s1->vec[2], seed, 0, 1, 2); + mld_polyeta_pack(s1_packed + 0 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[0]); + mld_polyeta_pack(s1_packed + 1 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[1]); + mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); +#endif /* MLD_CONFIG_PARAMETER_SET == 87 */ +#else /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && MLD_CONFIG_USE_X3 */ /* Batches keep the nonce grouping of the serial path; s2 and don't-care * outputs go to s1 slots that are yet to be sampled. */ #if MLD_CONFIG_PARAMETER_SET == 44 @@ -288,7 +348,7 @@ __contract__( mld_polyeta_pack(s1_packed + 2 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[2]); mld_polyeta_pack(s1_packed + 3 * MLDSA_POLYETA_PACKEDBYTES, &s1->vec[3]); #endif /* MLD_CONFIG_PARAMETER_SET == 87 */ -#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_USE_X3 */ } /** From 8dc139ba115535c6fd747388ac787438d712ce3b Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:15 +0000 Subject: [PATCH 5/8] test: Add an x3 test configuration and update component benchmarks Signed-off-by: Soumya Bhattacharya --- BIBLIOGRAPHY.md | 3 + test/bench/bench_components_mldsa.c | 105 +++- test/configs/configs.yml | 5 + test/configs/x3_keccak_config.h | 885 ++++++++++++++++++++++++++++ 4 files changed, 996 insertions(+), 2 deletions(-) create mode 100644 test/configs/x3_keccak_config.h diff --git a/BIBLIOGRAPHY.md b/BIBLIOGRAPHY.md index 33762b9be1..22724f97d9 100644 --- a/BIBLIOGRAPHY.md +++ b/BIBLIOGRAPHY.md @@ -117,6 +117,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) ### `FIPS202` @@ -184,6 +185,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) - [test/src/test_sign_hook.c](test/src/test_sign_hook.c) ### `FIPS204_UPDATES` @@ -225,6 +227,7 @@ source code and documentation. - [test/configs/serial_fips202_config.h](test/configs/serial_fips202_config.h) - [test/configs/test_alloc_config.h](test/configs/test_alloc_config.h) - [test/configs/test_sign_hook_config.h](test/configs/test_sign_hook_config.h) + - [test/configs/x3_keccak_config.h](test/configs/x3_keccak_config.h) - [test/src/test_sign_hook.c](test/src/test_sign_hook.c) ### `HYBRID` diff --git a/test/bench/bench_components_mldsa.c b/test/bench/bench_components_mldsa.c index 788e02b7b6..e8e7ee7de5 100644 --- a/test/bench/bench_components_mldsa.c +++ b/test/bench/bench_components_mldsa.c @@ -15,6 +15,10 @@ #include "../../mldsa/src/randombytes.h" #include "hal.h" +/* Batched Keccak permute declarations and lane/way constants for the + * keccakf1600x4_permute and keccakf1600x3_permute benchmarks. */ +#include "../../mldsa/src/fips202/keccakf1600.h" + #ifndef MLD_BENCHMARK_NWARMUP #define MLD_BENCHMARK_NWARMUP 50 #endif @@ -96,10 +100,29 @@ static int bench(void) BENCH("poly_uniform", mld_poly_uniform(&poly_out, seed)) } + /* poly_uniform_4x — the 4-way batched ExpandA (matrix) sampler; x4 + * counterpart to poly_uniform_3x below. Only benchmarked in the x4 build: + * under MLD_CONFIG_USE_X3 its Keccak falls back to a non-native path, so the + * number would not reflect the real x4 kernel. Guard also matches its + * declaration in poly.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_a0, poly_a1, poly_a2, poly_a3; + MLD_ALIGN uint8_t seed_ext4[4][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + CHECK(mld_randombytes((uint8_t *)seed_ext4, sizeof(seed_ext4)) == 0); + BENCH("poly_uniform_4x", mld_poly_uniform_4x(&poly_a0, &poly_a1, &poly_a2, + &poly_a3, seed_ext4)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) */ + BENCH("poly_caddq", mld_poly_caddq((mld_poly *)data0)); - /* poly_uniform_eta_4x — exercises rej_uniform_eta{2,4}_avx2_asm on x86 */ -#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + /* poly_uniform_eta_4x — exercises rej_uniform_eta{2,4}_avx2_asm on x86. + * Not benchmarked with MLD_CONFIG_USE_X3, where the AArch64 backend provides + * the x3 kernel instead and this row would measure a fallback. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ !defined(MLD_CONFIG_NO_KEYPAIR_API) { MLD_ALIGN mld_poly poly_eta0, poly_eta1, poly_eta2, poly_eta3; @@ -108,8 +131,86 @@ static int bench(void) mld_poly_uniform_eta_4x(&poly_eta0, &poly_eta1, &poly_eta2, &poly_eta3, (const uint8_t *)data0, 0, 1, 2, 3)) } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + !MLD_CONFIG_NO_KEYPAIR_API */ + + /* poly_uniform_gamma1_4x — ExpandMask x4 sampler. x4 build only (same reason + * as eta_4x above). Guard matches its declaration in poly_kl.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_g0, poly_g1, poly_g2, poly_g3; + BENCH("poly_uniform_gamma1_4x", + mld_poly_uniform_gamma1_4x(&poly_g0, &poly_g1, &poly_g2, &poly_g3, + (const uint8_t *)data0, 0, 1, 2, 3)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) */ + + /* keccakf1600x4_permute — raw 4-way permutation (four sequential Keccak + * states, one call); x4 counterpart to keccakf1600x3_permute below (per + * state: x4 value / 4 vs x3 value / 3). x4 build only (same reason as + * eta_4x above). Guard matches its declaration in keccakf1600.h. */ +#if !defined(MLD_CONFIG_USE_X3) && !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_REDUCE_RAM) || \ + defined(MLD_UNIT_TEST)) + { + MLD_ALIGN uint64_t x4_state[MLD_KECCAK_LANES * MLD_KECCAK_WAY]; + CHECK(mld_randombytes((uint8_t *)x4_state, sizeof(x4_state)) == 0); + BENCH("keccakf1600x4_permute", mld_keccakf1600x4_permute(x4_state)) + } +#endif /* !MLD_CONFIG_USE_X3 && !MLD_CONFIG_SERIAL_FIPS202_ONLY && \ + (!MLD_CONFIG_NO_KEYPAIR_API || !MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST) */ + +#if defined(MLD_CONFIG_USE_X3) + /* + * x3 (3-way) batched Keccak benchmarks, built with MLD_CONFIG_USE_X3: the raw + * permutation and the ExpandA, ExpandS and ExpandMask samplers, to compare + * with the x4 sampler rows above (per poly: x3 value / 3 vs x4 value / 4). + */ + + /* Raw x3 permutation (three sequential Keccak states, one call). */ + { + MLD_ALIGN uint64_t x3_state[MLD_KECCAK_LANES * MLD_KECCAK_WAY_X3]; + CHECK(mld_randombytes((uint8_t *)x3_state, sizeof(x3_state)) == 0); + BENCH("keccakf1600x3_permute", mld_keccakf1600x3_permute(x3_state)) + } + + /* ExpandA x3 sampler (SHAKE128, three matrix entries per call). */ + { + MLD_ALIGN mld_poly poly_a0, poly_a1, poly_a2; + MLD_ALIGN uint8_t seed_ext[3][MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + CHECK(mld_randombytes((uint8_t *)seed_ext, sizeof(seed_ext)) == 0); + BENCH("poly_uniform_3x", + mld_poly_uniform_3x(&poly_a0, &poly_a1, &poly_a2, seed_ext)) + } + + /* ExpandS x3 sampler (SHAKE256, three secret polynomials per call). */ +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) + { + MLD_ALIGN mld_poly poly_s0, poly_s1, poly_s2; + BENCH("poly_uniform_eta_3x", + mld_poly_uniform_eta_3x(&poly_s0, &poly_s1, &poly_s2, + (const uint8_t *)data0, 0, 1, 2)) + } #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && !MLD_CONFIG_NO_KEYPAIR_API */ + /* ExpandMask x3 sampler (SHAKE256, three gamma1 mask polynomials per call). + */ +#if !defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) && \ + (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) + { + MLD_ALIGN mld_poly poly_g0, poly_g1, poly_g2; + BENCH("poly_uniform_gamma1_3x", + mld_poly_uniform_gamma1_3x(&poly_g0, &poly_g1, &poly_g2, + (const uint8_t *)data0, 0, 1, 2)) + } +#endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY && (!MLD_CONFIG_REDUCE_RAM || \ + MLD_UNIT_TEST) */ +#endif /* MLD_CONFIG_USE_X3 */ + BENCH("poly_chknorm", chknorm_acc ^= mld_poly_chknorm((const mld_poly *)data0, MLDSA_GAMMA1 - MLDSA_BETA);) diff --git a/test/configs/configs.yml b/test/configs/configs.yml index 83f2d1985f..c0399d5ea0 100644 --- a/test/configs/configs.yml +++ b/test/configs/configs.yml @@ -28,6 +28,11 @@ configs: defines: MLD_CONFIG_SERIAL_FIPS202_ONLY: true + - path: test/configs/x3_keccak_config.h + description: "Test configuration with the 3-way (x3) batched Keccak path" + defines: + MLD_CONFIG_USE_X3: true + - path: test/configs/custom_randombytes_config.h description: "Test configuration with custom randombytes" defines: diff --git a/test/configs/x3_keccak_config.h b/test/configs/x3_keccak_config.h new file mode 100644 index 0000000000..6c4753702a --- /dev/null +++ b/test/configs/x3_keccak_config.h @@ -0,0 +1,885 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + * + * - [FIPS204_UPDATES] + * FIPS 204 Potential Updates (Errata) + * National Institute of Standards and Technology + * https://csrc.nist.gov/files/pubs/fips/204/final/docs/fips-204-potential-updates.xlsx + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +/* + * Test configuration: Test configuration with the 3-way (x3) batched Keccak + * path + * + * This configuration differs from the default mldsa/mldsa_native_config.h in + * the following places: + * - MLD_CONFIG_USE_X3 + */ + + +#ifndef MLD_CONFIG_H +#define MLD_CONFIG_H + +/** + * MLD_CONFIG_PARAMETER_SET + * + * Specifies the parameter set for ML-DSA + * - MLD_CONFIG_PARAMETER_SET=44 corresponds to ML-DSA-44 + * - MLD_CONFIG_PARAMETER_SET=65 corresponds to ML-DSA-65 + * - MLD_CONFIG_PARAMETER_SET=87 corresponds to ML-DSA-87 + * + * If you want to support multiple parameter sets, build the + * library multiple times and set MLD_CONFIG_MULTILEVEL_BUILD. + * See MLD_CONFIG_MULTILEVEL_BUILD for how to do this while + * minimizing code duplication. + * + * This can also be set using CFLAGS. + */ +#ifndef MLD_CONFIG_PARAMETER_SET +#define MLD_CONFIG_PARAMETER_SET \ + 44 /* Change this for different security strengths */ +#endif + +/** + * MLD_CONFIG_FILE + * + * If defined, this is a header that will be included instead + * of the default configuration file mldsa/mldsa_native_config.h. + * + * When you need to build mldsa-native in multiple configurations, + * using varying MLD_CONFIG_FILE can be more convenient + * than configuring everything through CFLAGS. + * + * To use, MLD_CONFIG_FILE _must_ be defined prior + * to the inclusion of any mldsa-native headers. For example, + * it can be set by passing `-DMLD_CONFIG_FILE="..."` + * on the command line. + */ +/* #define MLD_CONFIG_FILE "mldsa_native_config.h" */ + +/** + * MLD_CONFIG_NAMESPACE_PREFIX + * + * The prefix to use to namespace global symbols from mldsa/. + * + * In a multi-level build, level-dependent symbols will + * additionally be prefixed with the parameter set (44/65/87). + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_NAMESPACE_PREFIX) +#define MLD_CONFIG_NAMESPACE_PREFIX MLD_DEFAULT_NAMESPACE_PREFIX +#endif + +/** + * MLD_CONFIG_MULTILEVEL_BUILD + * + * Set this if the build is part of a multi-level build supporting + * multiple parameter sets. + * + * If you need only a single parameter set, keep this unset. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_BUILD */ + +/** + * MLD_CONFIG_EXTERNAL_API_QUALIFIER + * + * If set, this option provides an additional function + * qualifier to be added to declarations of mldsa-native's + * public API. + * + * The primary use case for this option are single-CU builds + * where the public API exposed by mldsa-native is wrapped by + * another API in the consuming application. In this case, + * even mldsa-native's public API can be marked `static`. + */ +/* #define MLD_CONFIG_EXTERNAL_API_QUALIFIER */ + +/** + * MLD_CONFIG_NO_KEYPAIR_API + * + * By default, mldsa-native includes support for generating key + * pairs. If you don't need this, set MLD_CONFIG_NO_KEYPAIR_API + * to exclude keypair, keypair_internal, + * pk_from_sk, and all internal APIs only needed by + * those functions. + */ +/* #define MLD_CONFIG_NO_KEYPAIR_API */ + +/** + * MLD_CONFIG_NO_SIGN_API + * + * By default, mldsa-native includes support for creating + * signatures. If you don't need this, set MLD_CONFIG_NO_SIGN_API + * to exclude signature, + * signature_extmu, signature_internal, + * signature_pre_hash_internal, + * signature_pre_hash_shake256, and all internal APIs + * only needed by those functions. + */ +/* #define MLD_CONFIG_NO_SIGN_API */ + +/** + * MLD_CONFIG_NO_VERIFY_API + * + * By default, mldsa-native includes support for verifying + * signatures. If you don't need this, set + * MLD_CONFIG_NO_VERIFY_API to exclude verify, + * verify_extmu, verify_internal, + * verify_pre_hash_internal, + * verify_pre_hash_shake256, and all internal APIs + * only needed by those functions. + */ +/* #define MLD_CONFIG_NO_VERIFY_API */ + +/** + * MLD_CONFIG_CORE_API_ONLY + * + * Set this to remove all public APIs except + * keypair_internal, signature_internal, + * and verify_internal. + */ +/* #define MLD_CONFIG_CORE_API_ONLY */ + +/** + * MLD_CONFIG_NO_RANDOMIZED_API + * + * If this option is set, mldsa-native will be built without the + * randomized API functions (keypair, + * signature, and signature_extmu). + * This allows users to build mldsa-native without providing a + * randombytes() implementation if they only need the + * internal deterministic API + * (keypair_internal, signature_internal). + * + * @note This option is incompatible with MLD_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires + * signature(). + */ +/* #define MLD_CONFIG_NO_RANDOMIZED_API */ + +/** + * MLD_CONFIG_CONSTANTS_ONLY + * + * If you only need the size constants (MLDSA_PUBLICKEYBYTES, etc.) + * but no function declarations, set MLD_CONFIG_CONSTANTS_ONLY. + * + * This only affects the public header mldsa_native.h, not + * the implementation. + */ +/* #define MLD_CONFIG_CONSTANTS_ONLY */ + +/** + * MLD_CONFIG_KEYGEN_PCT + * + * Compliance with @[FIPS140_3_IG, p.87] requires a + * Pairwise Consistency Test (PCT) to be carried out on a freshly + * generated keypair before it can be exported. + * + * Set this option if such a check should be implemented. + * In this case, keypair_internal and + * keypair will return MLD_ERR_PCT_FAIL if the + * PCT failed. + * + * @note This feature will drastically lower the performance of + * key generation. + * + * @note This option is incompatible with MLD_CONFIG_NO_SIGN_API + * and MLD_CONFIG_NO_VERIFY_API as the current PCT implementation + * requires signature() and verify(). + */ +/* #define MLD_CONFIG_KEYGEN_PCT */ + +/** + * MLD_CONFIG_CONTEXT_PARAMETER + * + * Set this to add a caller-supplied context parameter to the public API + * functions, which is then forwarded unchanged to the custom callbacks + * (allocation, and signing hooks below). + * + * When this option is set, every public API function gains a trailing + * parameter + * + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE context + * + * as its last argument; its type is configured via + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE (see below). mldsa-native treats this + * value as opaque: it never dereferences it and only passes it on to the + * configurable hook macros. It is meant to carry per-caller state -- e.g. a + * pointer to a memory pool for the allocation hooks, or the resume state for + * the signing hooks -- into those hooks. + * + * When this option is unset (the default), no extra parameter is added and + * the hook macros never receive a context argument. + * + * The hooks that receive the context are the allocation hooks (see + * MLD_CONFIG_CUSTOM_ALLOC_FREE) and the signing hooks (see + * MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH); each is documented with + * its own option below. + */ +/* #define MLD_CONFIG_CONTEXT_PARAMETER */ + +/** + * MLD_CONFIG_CONTEXT_PARAMETER_TYPE + * + * Set this to define the type of the context parameter added by + * MLD_CONFIG_CONTEXT_PARAMETER. It can be any C type usable as a function + * parameter, e.g. `void *` or a pointer to a caller-defined struct such as + * `struct my_ctx *`. + * + * This option must be defined if and only if MLD_CONFIG_CONTEXT_PARAMETER is + * defined; defining one without the other is a compile-time error. + */ +/* #define MLD_CONFIG_CONTEXT_PARAMETER_TYPE void* */ + +/** + * MLD_CONFIG_REDUCE_RAM + * + * Set this to reduce RAM usage. This trades memory for performance. + * + * For expected memory usage, see the MLD_TOTAL_ALLOC_* constants defined in + * mldsa_native.h. + * + * This option is useful for embedded systems with tight RAM constraints but + * relaxed performance requirements. + * + */ +/* #define MLD_CONFIG_REDUCE_RAM */ +/****************************************************************************** + * + * Build-only configuration options + * + * The remaining configurations only affect the build, not the API + * described in mldsa_native.h. + * + *****************************************************************************/ + +/** + * MLD_CONFIG_MULTILEVEL_WITH_SHARED + * + * This is for multi-level builds of mldsa-native only. If you + * need only a single parameter set, keep this unset. + * + * If this is set, all MLD_CONFIG_PARAMETER_SET-independent + * code will be included in the build, including code needed only + * for other parameter sets. + * + * Example: mld_polyw1_pack_88 is only needed for + * MLD_CONFIG_PARAMETER_SET == 44. Yet, if this option is set for a + * build with MLD_CONFIG_PARAMETER_SET == 65/87, it would be included. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_WITH_SHARED */ + +/** + * MLD_CONFIG_MULTILEVEL_NO_SHARED + * + * This is for multi-level builds of mldsa-native only. If you + * need only a single parameter set, keep this unset. + * + * If this is set, no MLD_CONFIG_PARAMETER_SET-independent code + * will be included in the build. + * + * To build mldsa-native with support for all parameter sets, + * build it three times -- once per parameter set -- and set the + * option MLD_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of + * them, and MLD_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLD_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +/** + * MLD_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * + * This is only relevant for single compilation unit (SCU) + * builds of mldsa-native. In this case, it determines whether + * directives defined in parameter-set-independent headers should + * be #undef'ined or not at the end of the SCU file. This is + * needed in multilevel builds. + * + * See examples/multilevel_build_native for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLD_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ + +/** + * MLD_CONFIG_USE_NATIVE_BACKEND_ARITH + * + * Determines whether a native arithmetic backend should be used. + * + * The arithmetic backend covers performance-critical functions + * such as the number-theoretic transform (NTT). + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the arithmetic backend to be used is + * determined by MLD_CONFIG_ARITH_BACKEND_FILE: If the latter is + * unset, the default backend for your target architecture + * will be used. If set, it must be the name of a backend metadata + * file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* #define MLD_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif + +/** + * MLD_CONFIG_ARITH_BACKEND_FILE + * + * The arithmetic backend to use. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option + * is ignored. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must + * either be undefined or the filename of an arithmetic backend. + * If unset, the default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLD_CONFIG_ARITH_BACKEND_FILE) +#define MLD_CONFIG_ARITH_BACKEND_FILE "native/meta.h" +#endif + +/** + * MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * + * Determines whether a native FIPS202 backend should be used. + * + * The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is + * the performance bottleneck of SHA3 and SHAKE. + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the FIPS202 backend to be used is + * determined by MLD_CONFIG_FIPS202_BACKEND_FILE: If the latter is + * unset, the default backend for your target architecture + * will be used. If set, it must be the name of a backend metadata + * file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#endif + +/** + * MLD_CONFIG_USE_X3 + * + * If set, the AArch64 FIPS202 backend uses the 3-way (x3) hybrid + * Keccak-f1600 kernel (two states on Neon using the Armv8.4-A SHA3 + * instructions, one state on scalar registers, single pass) instead of the + * default 4-way (x4) hybrid, and ExpandA, ExpandS and ExpandMask batch their + * XOF calls three at a time to match. The sampled values are identical to the + * default path; only the batching differs. + * + * This is intended for AArch64 cores that execute the SHA3 instructions on all + * SIMD pipes (e.g. Arm Neoverse V3). On Arm Neoverse V1 and V2, where it was + * tested, the default x4 hybrid is faster than x3. + * + * This can also be set using CFLAGS. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. + */ +#define MLD_CONFIG_USE_X3 + +/** + * MLD_CONFIG_FIPS202_BACKEND_FILE + * + * The FIPS-202 backend to use. + * + * If MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option + * must either be undefined or the filename of a FIPS202 backend. + * If unset, the default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLD_CONFIG_FIPS202_BACKEND_FILE) +#define MLD_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h" +#endif + +/** + * MLD_CONFIG_FIPS202_CUSTOM_HEADER + * + * Custom header to use for FIPS-202 + * + * This should only be set if you intend to use a custom + * FIPS-202 implementation, different from the one shipped + * with mldsa-native. + * + * If set, it must be the name of a file serving as the + * replacement for mldsa/src/fips202/fips202.h, and exposing + * the same API (see FIPS202.md). + */ +/* #define MLD_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLD_CONFIG_FIPS202X4_CUSTOM_HEADER + * + * Custom header to use for FIPS-202-X4 + * + * This should only be set if you intend to use a custom + * FIPS-202 implementation, different from the one shipped + * with mldsa-native. + * + * If set, it must be the name of a file serving as the + * replacement for mldsa/src/fips202/fips202x4.h, and exposing + * the same API (see FIPS202.md). + */ +/* #define MLD_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLD_CONFIG_CUSTOM_ZEROIZE + * + * In compliance with @[FIPS204, Section 3.6.3], mldsa-native zeroizes + * intermediate buffers before returning from function calls. By default, + * those buffers are allocated from the stack; if MLD_CONFIG_CUSTOM_ALLOC_FREE + * is set, they are (mostly -- few exceptions remain at present) allocated from + * the configured custom allocator. + * + * mldsa-native also zeroizes caller-owned output buffers as needed to uphold + * the API convention that outputs be either unmodified or zeroized upon + * failure. + * + * Set this option and define `mld_zeroize` if you want to use a custom + * method to zeroize intermediate and output buffers. + * + * The default implementation uses SecureZeroMemory on Windows and a + * memset + compiler barrier otherwise. If neither of those is available on + * the target platform, compilation will fail, and you will need to use + * MLD_CONFIG_CUSTOM_ZEROIZE to provide a custom implementation of + * `mld_zeroize()`. + * + * @warning + * The zeroization conducted by mldsa-native reduces the likelihood of data + * leaking on the stack or custom allocators, but it does not eliminate it. + * For example, the C standard makes no guarantee about where a compiler + * allocates local structures and whether/where it makes copies of them. + * Also, in addition to entire structures, there may also be potentially + * exploitable leakage of individual values on the stack. If you need + * bullet-proof zeroization of the stack, you need to consider additional + * measures instead of what this feature provides. In this case, you can + * set mld_zeroize to a no-op. Note that in this case you are also responsible + * for zeroizing output buffers upon failure. + */ +/* #define MLD_CONFIG_CUSTOM_ZEROIZE + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void mld_zeroize(void *ptr, size_t len) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_RANDOMBYTES + * + * mldsa-native does not provide a secure randombytes + * implementation. Such an implementation has to be provided by + * the consumer. + * + * If this option is not set, mldsa-native expects a function + * int randombytes(uint8_t *out, size_t outlen). + * + * Set this option and define `mld_randombytes` if you want to + * use a custom method to sample randombytes with a different name + * or signature. + */ +/* #define MLD_CONFIG_CUSTOM_RANDOMBYTES + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE int mld_randombytes(uint8_t *ptr, size_t len) + { + ... your implementation ... + return 0; + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_CAPABILITY_FUNC + * + * mldsa-native backends may rely on specific hardware features. + * Those backends will only be included in an mldsa-native build + * if support for the respective features is enabled at + * compile-time. However, when building for a heterogeneous set + * of CPUs to run the resulting binary/library on, feature + * detection at _runtime_ is needed to decide whether a backend + * can be used or not. + * + * Set this option and define `mld_sys_check_capability` if you + * want to use a custom method to dispatch between implementations. + * + * Return value 1 indicates that a capability is supported. + * Return value 0 indicates that a capability is not supported. + * + * If this option is not set, mldsa-native uses compile-time + * feature detection only to decide which backend to use. + * + * If you compile mldsa-native on a system with different + * capabilities than the system that the resulting binary/library + * will be run on, you must use this option. + */ +/* #define MLD_CONFIG_CUSTOM_CAPABILITY_FUNC + static MLD_INLINE int mld_sys_check_capability(mld_sys_cap cap) + { + ... your implementation ... + } +*/ + +/** + * MLD_CONFIG_CUSTOM_ALLOC_FREE + * + * Set this option and define `MLD_CUSTOM_ALLOC` and + * `MLD_CUSTOM_FREE` if you want to use custom allocation for + * large local structures or buffers. + * + * By default, all buffers/structures are allocated on the stack. + * If this option is set, most of them will be allocated via + * MLD_CUSTOM_ALLOC. + * + * Parameters to MLD_CUSTOM_ALLOC: + * - T* v: Target pointer to declare. + * - T: Type of structure to be allocated + * - N: Number of elements to be allocated. + * + * Parameters to MLD_CUSTOM_FREE: + * - T* v: Target pointer to free. May be NULL. + * - T: Type of structure to be freed. + * - N: Number of elements to be freed. + * + * @note Even if this option is set, some allocations further down + * the call stack will still be made from the stack. Those will + * likely be added to the scope of this option in the future. + * + * @note MLD_CUSTOM_ALLOC need not guarantee a successful + * allocation nor include error handling. Upon failure, the + * target pointer should simply be set to NULL. The calling + * code will handle this case and invoke MLD_CUSTOM_FREE. + */ +/* #define MLD_CONFIG_CUSTOM_ALLOC_FREE + #if !defined(__ASSEMBLER__) + #include + #define MLD_CUSTOM_ALLOC(v, T, N) \ + T* (v) = (T *)aligned_alloc(MLD_DEFAULT_ALIGN, \ + MLD_ALIGN_UP(sizeof(T) * (N))) + #define MLD_CUSTOM_FREE(v, T, N) free(v) + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_MEMCPY + * + * Set this option and define `mld_memcpy` if you want to + * use a custom method to copy memory instead of the standard + * library memcpy function. + * + * The custom implementation must have the same signature and + * behavior as the standard memcpy function: + * void *mld_memcpy(void *dest, const void *src, size_t n) + */ +/* #define MLD_CONFIG_CUSTOM_MEMCPY + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void *mld_memcpy(void *dest, const void *src, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_CUSTOM_MEMSET + * + * Set this option and define `mld_memset` if you want to + * use a custom method to set memory instead of the standard + * library memset function. + * + * The custom implementation must have the same signature and + * behavior as the standard memset function: + * void *mld_memset(void *s, int c, size_t n) + */ +/* #define MLD_CONFIG_CUSTOM_MEMSET + #if !defined(__ASSEMBLER__) + #include + #include "src/src.h" + static MLD_INLINE void *mld_memset(void *s, int c, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLD_CONFIG_INTERNAL_API_QUALIFIER + * + * If set, this option provides an additional qualifier + * to be added to declarations of internal API functions and data. + * + * The primary use case for this option are single-CU builds, + * in which case this option can be set to `static`. + */ +/* #define MLD_CONFIG_INTERNAL_API_QUALIFIER */ + +/** + * MLD_CONFIG_CT_TESTING_ENABLED + * + * If set, mldsa-native annotates data as secret / public using + * valgrind's annotations VALGRIND_MAKE_MEM_UNDEFINED and + * VALGRIND_MAKE_MEM_DEFINED, enabling various checks for secret- + * dependent control flow of variable time execution (depending + * on the exact version of valgrind installed). + */ +/* #define MLD_CONFIG_CT_TESTING_ENABLED */ + +/** + * MLD_CONFIG_NO_ASM + * + * If this option is set, mldsa-native will be built without + * use of native code or inline assembly. + * + * By default, inline assembly is used to implement value barriers. + * Without inline assembly, mldsa-native will use a global volatile + * 'opt blocker' instead; see ct.h. + * + * Inline assembly is also used to implement a secure zeroization + * function on non-Windows platforms. If this option is set and + * the target platform is not Windows, you MUST set + * MLD_CONFIG_CUSTOM_ZEROIZE and provide a custom zeroization + * function. + * + * If this option is set, MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 and + * MLD_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no + * native backends will be used. + */ +/* #define MLD_CONFIG_NO_ASM */ + +/** + * MLD_CONFIG_NO_ASM_VALUE_BARRIER + * + * If this option is set, mldsa-native will be built without + * use of native code or inline assembly for value barriers. + * + * By default, inline assembly (if available) is used to implement + * value barriers. + * Without inline assembly, mldsa-native will use a global volatile + * 'opt blocker' instead; see ct.h. + */ +/* #define MLD_CONFIG_NO_ASM_VALUE_BARRIER */ + +/** + * MLD_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + * + * If this option is set, the user must provide a runtime + * function `static inline int mld_break_pct() { ... }` to + * indicate whether the PCT should be made fail. + * + * This option only has an effect if MLD_CONFIG_KEYGEN_PCT is set. + */ +/* #define MLD_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + #if !defined(__ASSEMBLER__) + #include "src/src.h" + static MLD_INLINE int mld_break_pct(void) + { + ... return 0/1 depending on whether PCT should be broken ... + } + #endif +*/ + +/** + * MLD_CONFIG_MAX_SIGNING_ATTEMPTS + * + * Upper bound on the number of rejection-sampling iterations + * performed by ML-DSA signing (@[FIPS204, Algorithm 7]). + * + * If a valid signature is not produced within this many + * attempts, signing returns MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED. + * This is useful in timing-sensitive environments that + * require a deterministic worst-case bound on signing time. + * + * For FIPS 204 compliance, this value MUST be at least 821, + * cf. @[FIPS204, Appendix C] and @[FIPS204_UPDATES], which is + * chosen so that the signing failure rate is < 2^{-256}. + * + * Default: Largest possible value before internal counters + * would overflow. This is larger than the FIPS204 bound. + * + * In particular, in the default configuration, the signing + * failure rate is < 2^{-256}. + */ +/* #define MLD_CONFIG_MAX_SIGNING_ATTEMPTS 821 */ + +/** + * MLD_CONFIG_SERIAL_FIPS202_ONLY + * + * Set this to use a FIPS202 implementation with global state + * that supports only one active Keccak computation at a time + * (e.g. some hardware accelerators). + * + * If this option is set, ML-DSA will use FIPS202 operations + * serially, ensuring that only one SHAKE context is active + * at any given time. + * + * This allows offloading Keccak computations to a hardware + * accelerator that holds only a single Keccak state locally, + * rather than requiring support for multiple concurrent + * Keccak states. + * + * @note Depending on the target CPU, this may reduce + * performance when using software FIPS202 implementations. + * Only enable this when you have to. + */ +/* #define MLD_CONFIG_SERIAL_FIPS202_ONLY */ + +/** + * Signing hooks: MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH + * + * Three optional, independent hooks into the ML-DSA signing rejection-sampling + * loop. Each is enabled by defining the matching option, in which case the + * integration must provide the corresponding function. If a hook needs + * per-operation state, enable MLD_CONFIG_CONTEXT_PARAMETER; the context is then + * appended as the last argument. + * + * @warning This feature is experimental. Its scope, configuration and + * function signatures may change at any time, including after v2. + * + * Enabling any of the hooks requires MLD_CONFIG_NO_RANDOMIZED_API (restricting + * the public API to deterministic operations). This is because the restartable + * signing as enabled by the signing hooks only produces the uninterrupted + * signature when the randomness is fixed across calls. A logging-only use + * (attempt always returns 0; resume/finish merely observe) would be safe with + * the randomized API too, but for now the requirement is imposed uniformly on + * all three hooks. + * + * Note: Randomized signing is a shim wrapper around deterministic signing, and + * all helper functions you need to build it are exposed publicly. Thus, if you + * need a restartable, randomized signing operation, you can build your own by + * replicating the logic and adding the RNG seed to the restart context. In this + * case, please also consider letting the mldsa-native maintainers know of your + * need for randomized, restartable signing, so the feature can be appropriately + * prioritized. + * + * - MLD_CONFIG_SIGN_HOOK_ATTEMPT: int mld_sign_hook_attempt(attempt[, ctxt]) + * Called before each attempt. Returns 0 to proceed, or non-zero to pause: + * signing then returns MLD_ERR_SIGNING_PAUSED with `attempt` as the resume + * point (needs MLD_CONFIG_SIGN_HOOK_RESUME to resume; otherwise just aborts). + * Always returning 0 makes it a logging/benchmarking hook. + * + * - MLD_CONFIG_SIGN_HOOK_RESUME: uint16_t mld_sign_hook_resume([ctxt]) + * Returns the attempt to resume from (0 for a fresh operation), i.e. the one + * recorded when a previous call paused. + * + * - MLD_CONFIG_SIGN_HOOK_FINISH: void mld_sign_hook_finish(attempt[, ctxt]) + * Called on success with the succeeding attempt. Observe-only. + * + * When an option is unset, the hook is a no-op (resume to 0, attempt proceeds), + * i.e. ordinary one-shot signing. + * + * Independent of MLD_CONFIG_MAX_SIGNING_ATTEMPTS, which is a static upper bound + * on the number of signing attempts. + * + * See test/src/test_sign_hook.c for a worked example using all three. + */ +/* #define MLD_CONFIG_SIGN_HOOK_RESUME + #define MLD_CONFIG_SIGN_HOOK_ATTEMPT + #define MLD_CONFIG_SIGN_HOOK_FINISH + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLD_INLINE uint16_t mld_sign_hook_resume(void) + { + ... return the attempt to resume from ... + } + static MLD_INLINE int mld_sign_hook_attempt(uint16_t attempt) + { + ... return non-zero to pause here; for resume, store attempt ... + return 0; + } + static MLD_INLINE void mld_sign_hook_finish(uint16_t attempt) + { + ... mark the operation complete (attempt = successful attempt) ... + } + #endif +*/ + +/************************* Config internals ********************************/ + +/* Default namespace + * + * Don't change this. If you need a different namespace, re-define + * MLD_CONFIG_NAMESPACE_PREFIX above instead, and remove the following. + * + * The default MLDSA namespace is + * + * PQCP_MLDSA_NATIVE_MLDSA_ + * + * e.g., PQCP_MLDSA_NATIVE_MLDSA44_ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_BUILD) +/* In a multi-level build the parameter set is appended by the namespacing + * machinery, so the default prefix must not embed it. */ +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA +#elif MLD_CONFIG_PARAMETER_SET == 44 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA44 +#elif MLD_CONFIG_PARAMETER_SET == 65 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA65 +#elif MLD_CONFIG_PARAMETER_SET == 87 +#define MLD_DEFAULT_NAMESPACE_PREFIX PQCP_MLDSA_NATIVE_MLDSA87 +#endif + +#endif /* !MLD_CONFIG_H */ From f71add12bc8cf098ba2b43e3d51105c3eaf1dc07 Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:15 +0000 Subject: [PATCH 6/8] CBMC: Add proofs for the 3-way batched Keccak and samplers Mirror the existing proofs for the x3 primitives, samplers and callers, built with MLD_CONFIG_USE_X3. Signed-off-by: Soumya Bhattacharya --- proofs/cbmc/dummy_backend_fips202_x3.h | 14 ++++ proofs/cbmc/keccak_absorb_once_x3/Makefile | 57 ++++++++++++++++ .../keccak_absorb_once_x3_harness.c | 20 ++++++ .../Makefile | 56 +++++++++++++++ ...ative_aarch64_v84a_scalar_hybrid_harness.c | 12 ++++ proofs/cbmc/keccak_squeezeblocks_x3/Makefile | 57 ++++++++++++++++ .../keccak_squeezeblocks_x3_harness.c | 18 +++++ .../cbmc/keccakf1600x3_extract_bytes/Makefile | 56 +++++++++++++++ .../keccakf1600x3_extract_bytes_harness.c | 13 ++++ .../keccakf1600x3_extract_bytes_c/Makefile | 56 +++++++++++++++ .../keccakf1600x3_extract_bytes_c_harness.c | 18 +++++ proofs/cbmc/keccakf1600x3_permute/Makefile | 56 +++++++++++++++ .../keccakf1600x3_permute_harness.c | 10 +++ .../keccakf1600x3_permute_native/Makefile | 56 +++++++++++++++ .../keccakf1600x3_permute_native_harness.c | 10 +++ proofs/cbmc/keccakf1600x3_xor_bytes/Makefile | 56 +++++++++++++++ .../keccakf1600x3_xor_bytes_harness.c | 13 ++++ .../cbmc/keccakf1600x3_xor_bytes_c/Makefile | 56 +++++++++++++++ .../keccakf1600x3_xor_bytes_c_harness.c | 18 +++++ proofs/cbmc/poly_uniform_3x/Makefile | 63 +++++++++++++++++ .../poly_uniform_3x/poly_uniform_3x_harness.c | 16 +++++ proofs/cbmc/poly_uniform_eta_3x/Makefile | 58 ++++++++++++++++ .../poly_uniform_eta_3x_harness.c | 12 ++++ proofs/cbmc/poly_uniform_gamma1_3x/Makefile | 63 +++++++++++++++++ .../poly_uniform_gamma1_3x_harness.c | 14 ++++ proofs/cbmc/polyvec_matrix_expand_x3/Makefile | 62 +++++++++++++++++ .../polyvec_matrix_expand_x3_harness.c | 12 ++++ .../cbmc/polyvecl_uniform_gamma1_x3/Makefile | 68 +++++++++++++++++++ .../polyvecl_uniform_gamma1_x3_harness.c | 15 ++++ proofs/cbmc/sample_pack_s1_s2_x3/Makefile | 56 +++++++++++++++ .../sample_pack_s1_s2_x3_harness.c | 19 ++++++ proofs/cbmc/shake128x3_absorb_once/Makefile | 61 +++++++++++++++++ .../shake128x3_absorb_once_harness.c | 16 +++++ proofs/cbmc/shake128x3_squeezeblocks/Makefile | 61 +++++++++++++++++ .../shake128x3_squeezeblocks_harness.c | 16 +++++ proofs/cbmc/shake256x3_absorb_once/Makefile | 56 +++++++++++++++ .../shake256x3_absorb_once_harness.c | 14 ++++ proofs/cbmc/shake256x3_squeezeblocks/Makefile | 56 +++++++++++++++ .../shake256x3_squeezeblocks_harness.c | 14 ++++ scripts/check-contracts | 2 + 40 files changed, 1406 insertions(+) create mode 100644 proofs/cbmc/dummy_backend_fips202_x3.h create mode 100644 proofs/cbmc/keccak_absorb_once_x3/Makefile create mode 100644 proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c create mode 100644 proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile create mode 100644 proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c create mode 100644 proofs/cbmc/keccak_squeezeblocks_x3/Makefile create mode 100644 proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_extract_bytes/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_permute/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_permute_native/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_xor_bytes/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c create mode 100644 proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile create mode 100644 proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c create mode 100644 proofs/cbmc/poly_uniform_3x/Makefile create mode 100644 proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c create mode 100644 proofs/cbmc/poly_uniform_eta_3x/Makefile create mode 100644 proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c create mode 100644 proofs/cbmc/poly_uniform_gamma1_3x/Makefile create mode 100644 proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c create mode 100644 proofs/cbmc/polyvec_matrix_expand_x3/Makefile create mode 100644 proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c create mode 100644 proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile create mode 100644 proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c create mode 100644 proofs/cbmc/sample_pack_s1_s2_x3/Makefile create mode 100644 proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c create mode 100644 proofs/cbmc/shake128x3_absorb_once/Makefile create mode 100644 proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c create mode 100644 proofs/cbmc/shake128x3_squeezeblocks/Makefile create mode 100644 proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c create mode 100644 proofs/cbmc/shake256x3_absorb_once/Makefile create mode 100644 proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c create mode 100644 proofs/cbmc/shake256x3_squeezeblocks/Makefile create mode 100644 proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c diff --git a/proofs/cbmc/dummy_backend_fips202_x3.h b/proofs/cbmc/dummy_backend_fips202_x3.h new file mode 100644 index 0000000000..24a4db5e71 --- /dev/null +++ b/proofs/cbmc/dummy_backend_fips202_x3.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DUMMY_FIPS202X3_BACKEND_H +#define MLD_DUMMY_FIPS202X3_BACKEND_H + + +#define MLD_USE_NATIVE_FIPS202_X3 + +#include "../../mldsa/src/fips202/native/api.h" + +#endif /* !MLD_DUMMY_FIPS202X3_BACKEND_H */ diff --git a/proofs/cbmc/keccak_absorb_once_x3/Makefile b/proofs/cbmc/keccak_absorb_once_x3/Makefile new file mode 100644 index 0000000000..2bcf0842c0 --- /dev/null +++ b/proofs/cbmc/keccak_absorb_once_x3/Makefile @@ -0,0 +1,57 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_absorb_once_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_absorb_once_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS+=mld_keccakf1600x3_xor_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccak_absorb_once_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c b/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c new file mode 100644 index 0000000000..3d60cc285a --- /dev/null +++ b/proofs/cbmc/keccak_absorb_once_x3/keccak_absorb_once_x3_harness.c @@ -0,0 +1,20 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void mld_keccak_absorb_once_x3(uint64_t *s, unsigned r, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + size_t inlen, uint8_t p); + +void harness(void) +{ + uint64_t *s; + unsigned r; + const uint8_t *in0, *in1, *in2; + size_t inlen; + uint8_t p; + mld_keccak_absorb_once_x3(s, r, in0, in1, in2, inlen, p); +} diff --git a/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile new file mode 100644 index 0000000000..573724cadd --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mlkem-native project authors +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid + +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 -DMLD_CONFIG_FIPS202_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/fips202/native/aarch64/x3_v84a_scalar.h\"" -DMLD_CHECK_APIS -D__ARM_FEATURE_SHA3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/native/aarch64/src/keccakf1600_round_constants.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_native +USE_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_v84a_scalar_hybrid_aarch64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c new file mode 100644 index 0000000000..ccd48eab0a --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid/keccak_f1600_x3_native_aarch64_v84a_scalar_hybrid_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +int mld_keccak_f1600_x3_native(uint64_t *state); + +void harness(void) +{ + uint64_t *s; + int t = mld_keccak_f1600_x3_native(s); +} diff --git a/proofs/cbmc/keccak_squeezeblocks_x3/Makefile b/proofs/cbmc/keccak_squeezeblocks_x3/Makefile new file mode 100644 index 0000000000..bd1c4e804d --- /dev/null +++ b/proofs/cbmc/keccak_squeezeblocks_x3/Makefile @@ -0,0 +1,57 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_squeezeblocks_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_squeezeblocks_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes +USE_FUNCTION_CONTRACTS+=mld_keccakf1600x3_permute +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccak_squeezeblocks_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c b/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c new file mode 100644 index 0000000000..5e37766f37 --- /dev/null +++ b/proofs/cbmc/keccak_squeezeblocks_x3/keccak_squeezeblocks_x3_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void mld_keccak_squeezeblocks_x3(uint8_t *out0, uint8_t *out1, uint8_t *out2, + size_t nblocks, uint64_t *s, unsigned r); + +void harness(void) +{ + uint8_t *out0, *out1, *out2; + size_t nblocks; + uint64_t *s; + unsigned r; + mld_keccak_squeezeblocks_x3(out0, out1, out2, nblocks, s, r); +} diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile b/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile new file mode 100644 index 0000000000..b69eb38433 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_extract_bytes_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_extract_bytes + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes_c +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccakf1600x3_extract_bytes + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c b/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c new file mode 100644 index 0000000000..fcd2eab76f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes/keccakf1600x3_extract_bytes_harness.c @@ -0,0 +1,13 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *state; + unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_extract_bytes(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile b/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile new file mode 100644 index 0000000000..dda7a71110 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes_c/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_extract_bytes_c_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_extract_bytes_c + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_extract_bytes_c +USE_FUNCTION_CONTRACTS=mld_keccakf1600_extract_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_extract_bytes_c + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c b/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c new file mode 100644 index 0000000000..ae2c7d0d99 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_extract_bytes_c/keccakf1600x3_extract_bytes_c_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void mld_keccakf1600x3_extract_bytes_c(uint64_t *state, unsigned char *data0, + unsigned char *data1, + unsigned char *data2, unsigned offset, + unsigned length); + +void harness(void) +{ + uint64_t *state; + unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_extract_bytes_c(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_permute/Makefile b/proofs/cbmc/keccakf1600x3_permute/Makefile new file mode 100644 index 0000000000..54af6a8a23 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_permute_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_permute + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS=mld_keccakf1600_permute +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_permute + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c b/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c new file mode 100644 index 0000000000..62aeefacc0 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute/keccakf1600x3_permute_harness.c @@ -0,0 +1,10 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *s; + mld_keccakf1600x3_permute(s); +} diff --git a/proofs/cbmc/keccakf1600x3_permute_native/Makefile b/proofs/cbmc/keccakf1600x3_permute_native/Makefile new file mode 100644 index 0000000000..af20c36e8b --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute_native/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_permute_native_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_permute_native + +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 -DMLD_CONFIG_FIPS202_BACKEND_FILE="\"dummy_backend_fips202_x3.h\"" -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_permute +USE_FUNCTION_CONTRACTS=mld_keccak_f1600_x3_native +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_permute_native + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c b/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c new file mode 100644 index 0000000000..62aeefacc0 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_permute_native/keccakf1600x3_permute_native_harness.c @@ -0,0 +1,10 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *s; + mld_keccakf1600x3_permute(s); +} diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile b/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile new file mode 100644 index 0000000000..095a7f4e81 --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_xor_bytes_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_xor_bytes + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes +USE_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes_c +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_keccakf1600x3_xor_bytes + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c b/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c new file mode 100644 index 0000000000..68a9ced14f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes/keccakf1600x3_xor_bytes_harness.c @@ -0,0 +1,13 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void harness(void) +{ + uint64_t *state; + const unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_xor_bytes(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile b/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile new file mode 100644 index 0000000000..561a01c4eb --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes_c/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccakf1600x3_xor_bytes_c_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccakf1600x3_xor_bytes_c + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/keccakf1600.c + +CHECK_FUNCTION_CONTRACTS=mld_keccakf1600x3_xor_bytes_c +USE_FUNCTION_CONTRACTS=mld_keccakf1600_xor_bytes +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_keccakf1600x3_xor_bytes_c + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c b/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c new file mode 100644 index 0000000000..3f9e1dca6f --- /dev/null +++ b/proofs/cbmc/keccakf1600x3_xor_bytes_c/keccakf1600x3_xor_bytes_c_harness.c @@ -0,0 +1,18 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include + +void mld_keccakf1600x3_xor_bytes_c(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, unsigned offset, + unsigned length); + +void harness(void) +{ + uint64_t *state; + const unsigned char *data0, *data1, *data2; + unsigned offset; + unsigned length; + mld_keccakf1600x3_xor_bytes_c(state, data0, data1, data2, offset, length); +} diff --git a/proofs/cbmc/poly_uniform_3x/Makefile b/proofs/cbmc/poly_uniform_3x/Makefile new file mode 100644 index 0000000000..684001abfc --- /dev/null +++ b/proofs/cbmc/poly_uniform_3x/Makefile @@ -0,0 +1,63 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_3x +USE_FUNCTION_CONTRACTS=mld_shake128x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake128x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_rej_uniform +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c b/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c new file mode 100644 index 0000000000..b1162fe39c --- /dev/null +++ b/proofs/cbmc/poly_uniform_3x/poly_uniform_3x_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_poly *r0; + mld_poly *r1; + mld_poly *r2; + uint8_t (*seed)[MLD_ALIGN_UP(MLDSA_SEEDBYTES + 2)]; + + mld_poly_uniform_3x(r0, r1, r2, seed); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/poly_uniform_eta_3x/Makefile b/proofs/cbmc/poly_uniform_eta_3x/Makefile new file mode 100644 index 0000000000..99c8697c1e --- /dev/null +++ b/proofs/cbmc/poly_uniform_eta_3x/Makefile @@ -0,0 +1,58 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_eta_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_eta_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_eta_3x +USE_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_rej_eta +USE_FUNCTION_CONTRACTS+=mld_zeroize +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_eta_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c b/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c new file mode 100644 index 0000000000..8b263fd19a --- /dev/null +++ b/proofs/cbmc/poly_uniform_eta_3x/poly_uniform_eta_3x_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly_kl.h" + +void harness(void) +{ + mld_poly *r0, *r1, *r2; + const uint8_t *seed; + uint8_t n0, n1, n2; + mld_poly_uniform_eta_3x(r0, r1, r2, seed, n0, n1, n2); +} diff --git a/proofs/cbmc/poly_uniform_gamma1_3x/Makefile b/proofs/cbmc/poly_uniform_gamma1_3x/Makefile new file mode 100644 index 0000000000..c65ae6c6d5 --- /dev/null +++ b/proofs/cbmc/poly_uniform_gamma1_3x/Makefile @@ -0,0 +1,63 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_uniform_gamma1_3x_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_uniform_gamma1_3x + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_poly_uniform_gamma1_3x +USE_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS+=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS+=mld_polyz_unpack +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_uniform_gamma1_3x + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c b/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c new file mode 100644 index 0000000000..88cb97a239 --- /dev/null +++ b/proofs/cbmc/poly_uniform_gamma1_3x/poly_uniform_gamma1_3x_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "poly_kl.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_poly *r0, *r1, *r2; + const uint8_t *seed; + uint16_t n0, n1, n2; + mld_poly_uniform_gamma1_3x(r0, r1, r2, seed, n0, n1, n2); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/polyvec_matrix_expand_x3/Makefile b/proofs/cbmc/polyvec_matrix_expand_x3/Makefile new file mode 100644 index 0000000000..87fdc556c3 --- /dev/null +++ b/proofs/cbmc/polyvec_matrix_expand_x3/Makefile @@ -0,0 +1,62 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyvec_matrix_expand_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyvec_matrix_expand_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/polyvec_lazy.c + +CHECK_FUNCTION_CONTRACTS=mld_polyvec_matrix_expand +ifdef MLD_CONFIG_REDUCE_RAM +USE_FUNCTION_CONTRACTS= +else +USE_FUNCTION_CONTRACTS=mld_poly_uniform_3x +USE_FUNCTION_CONTRACTS+=mld_poly_permute_bitrev_to_custom_optional +USE_FUNCTION_CONTRACTS+=mld_polymat_expand_entry +USE_FUNCTION_CONTRACTS+=mld_zeroize +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3_no_bv_extract +CBMC_DEFAULT_CONFIG = LP64:z3_no_bv_extract + +FUNCTION_NAME = polyvec_matrix_expand_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 12 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c b/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c new file mode 100644 index 0000000000..586ecbe493 --- /dev/null +++ b/proofs/cbmc/polyvec_matrix_expand_x3/polyvec_matrix_expand_x3_harness.c @@ -0,0 +1,12 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "polyvec_lazy.h" + +void harness(void) +{ + mld_polymat *mat; + uint8_t *rho; + + mld_polyvec_matrix_expand(mat, rho); +} diff --git a/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile b/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile new file mode 100644 index 0000000000..a4a6227894 --- /dev/null +++ b/proofs/cbmc/polyvecl_uniform_gamma1_x3/Makefile @@ -0,0 +1,68 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyvecl_uniform_gamma1_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyvecl_uniform_gamma1_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/polyvec.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_polyvecl_uniform_gamma1 +USE_FUNCTION_CONTRACTS=mld_poly_uniform_gamma1_3x +# L=5 (ML-DSA-65) uses a zeroized scratch lane; L=4 and L=7 sample one +# polynomial serially. +ifeq ($(MLD_CONFIG_PARAMETER_SET),65) +USE_FUNCTION_CONTRACTS+=mld_zeroize +else +USE_FUNCTION_CONTRACTS+=mld_poly_uniform_gamma1 +endif +endif + +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = polyvecl_uniform_gamma1_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 9 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c b/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c new file mode 100644 index 0000000000..c8e4a799cc --- /dev/null +++ b/proofs/cbmc/polyvecl_uniform_gamma1_x3/polyvecl_uniform_gamma1_x3_harness.c @@ -0,0 +1,15 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "polyvec.h" + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_polyvecl *v; + const uint8_t *seed; + uint16_t kappa; + + mld_polyvecl_uniform_gamma1(v, seed, kappa); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/sample_pack_s1_s2_x3/Makefile b/proofs/cbmc/sample_pack_s1_s2_x3/Makefile new file mode 100644 index 0000000000..d798a50787 --- /dev/null +++ b/proofs/cbmc/sample_pack_s1_s2_x3/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = sample_pack_s1_s2_x3_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = sample_pack_s1_s2_x3 + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/sign.c + +CHECK_FUNCTION_CONTRACTS=mld_sample_pack_s1_s2 +USE_FUNCTION_CONTRACTS=mld_poly_uniform_eta_3x +USE_FUNCTION_CONTRACTS+=mld_polyeta_pack +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:z3 +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = mld_sample_pack_s1_s2_x3 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c b/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c new file mode 100644 index 0000000000..fa13980785 --- /dev/null +++ b/proofs/cbmc/sample_pack_s1_s2_x3/sample_pack_s1_s2_x3_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include "sign.h" + +static void mld_sample_pack_s1_s2( + uint8_t s1_packed[MLDSA_L * MLDSA_POLYETA_PACKEDBYTES], + uint8_t s2_packed[MLDSA_K * MLDSA_POLYETA_PACKEDBYTES], mld_polyvecl *s1, + const uint8_t seed[MLDSA_CRHBYTES]); + +void harness(void) +{ + uint8_t *s1_packed; + uint8_t *s2_packed; + uint8_t *seed; + mld_polyvecl *s1; + + mld_sample_pack_s1_s2(s1_packed, s2_packed, s1, seed); +} diff --git a/proofs/cbmc/shake128x3_absorb_once/Makefile b/proofs/cbmc/shake128x3_absorb_once/Makefile new file mode 100644 index 0000000000..7108e00d04 --- /dev/null +++ b/proofs/cbmc/shake128x3_absorb_once/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake128x3_absorb_once_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake128x3_absorb_once + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_shake128x3_absorb_once +USE_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake128x3_absorb_once + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c b/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c new file mode 100644 index 0000000000..85809eb70c --- /dev/null +++ b/proofs/cbmc/shake128x3_absorb_once/shake128x3_absorb_once_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + mld_shake128x3ctx *state; + const uint8_t *in0, *in1, *in2; + size_t inlen; + mld_shake128x3_absorb_once(state, in0, in1, in2, inlen); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/shake128x3_squeezeblocks/Makefile b/proofs/cbmc/shake128x3_squeezeblocks/Makefile new file mode 100644 index 0000000000..20fa7d47cf --- /dev/null +++ b/proofs/cbmc/shake128x3_squeezeblocks/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake128x3_squeezeblocks_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake128x3_squeezeblocks + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +ifdef MLD_CONFIG_REDUCE_RAM +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS= +else +CHECK_FUNCTION_CONTRACTS=mld_shake128x3_squeezeblocks +USE_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake128x3_squeezeblocks + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c b/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c new file mode 100644 index 0000000000..a7e6746760 --- /dev/null +++ b/proofs/cbmc/shake128x3_squeezeblocks/shake128x3_squeezeblocks_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ +#if !defined(MLD_CONFIG_REDUCE_RAM) + uint8_t *out0, *out1, *out2; + size_t nblocks; + mld_shake128x3ctx *state; + mld_shake128x3_squeezeblocks(out0, out1, out2, nblocks, state); +#endif /* !MLD_CONFIG_REDUCE_RAM */ +} diff --git a/proofs/cbmc/shake256x3_absorb_once/Makefile b/proofs/cbmc/shake256x3_absorb_once/Makefile new file mode 100644 index 0000000000..ac3be69454 --- /dev/null +++ b/proofs/cbmc/shake256x3_absorb_once/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake256x3_absorb_once_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake256x3_absorb_once + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_shake256x3_absorb_once +USE_FUNCTION_CONTRACTS=mld_keccak_absorb_once_x3 +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake256x3_absorb_once + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c b/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c new file mode 100644 index 0000000000..f84a1242a4 --- /dev/null +++ b/proofs/cbmc/shake256x3_absorb_once/shake256x3_absorb_once_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ + mld_shake256x3ctx *state; + const uint8_t *in0, *in1, *in2; + size_t inlen; + mld_shake256x3_absorb_once(state, in0, in1, in2, inlen); +} diff --git a/proofs/cbmc/shake256x3_squeezeblocks/Makefile b/proofs/cbmc/shake256x3_squeezeblocks/Makefile new file mode 100644 index 0000000000..75efcf07cc --- /dev/null +++ b/proofs/cbmc/shake256x3_squeezeblocks/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) The mldsa-native project authors +# Copyright (c) The mlkem-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = shake256x3_squeezeblocks_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = shake256x3_squeezeblocks + +DEFINES += -DMLD_CONFIG_USE_X3 +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/fips202x3.c + +CHECK_FUNCTION_CONTRACTS=mld_shake256x3_squeezeblocks +USE_FUNCTION_CONTRACTS=mld_keccak_squeezeblocks_x3 +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = mld_shake256x3_squeezeblocks + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mlkem/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mlkem/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c b/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c new file mode 100644 index 0000000000..b6cf9a778a --- /dev/null +++ b/proofs/cbmc/shake256x3_squeezeblocks/shake256x3_squeezeblocks_harness.c @@ -0,0 +1,14 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include +#include + +void harness(void) +{ + uint8_t *out0, *out1, *out2; + size_t nblocks; + mld_shake256x3ctx *state; + mld_shake256x3_squeezeblocks(out0, out1, out2, nblocks, state); +} diff --git a/scripts/check-contracts b/scripts/check-contracts index e1dcc19b27..f37005658f 100755 --- a/scripts/check-contracts +++ b/scripts/check-contracts @@ -167,7 +167,9 @@ def check_contracts(): DECREASES_EXCEPTIONS = { ("mldsa/src/poly.c", "mld_poly_uniform"): 1, ("mldsa/src/poly.c", "mld_poly_uniform_4x"): 1, + ("mldsa/src/poly.c", "mld_poly_uniform_3x"): 1, ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta_4x"): 1, + ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta_3x"): 1, ("mldsa/src/poly_kl.c", "mld_poly_uniform_eta"): 1, ("mldsa/src/poly_kl.c", "mld_poly_challenge"): 1, } From c2d8b2ab88edc229b8318898b3348b54ef587ab4 Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Sat, 17 Oct 2026 03:57:15 +0000 Subject: [PATCH 7/8] CI: Test the x3 configuration Run the x3 config variation and the x3_v84a_scalar backend tests, and test the regenerated x3 kernel in the SLOTHY job. Signed-off-by: Soumya Bhattacharya --- .github/actions/config-variations/action.yml | 36 +++++++++++++++++++- .github/workflows/ci.yml | 7 ++-- .github/workflows/ci_ec2_reusable.yml | 2 ++ scripts/tests | 1 + 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.github/actions/config-variations/action.yml b/.github/actions/config-variations/action.yml index f38fc08134..73359dcf09 100644 --- a/.github/actions/config-variations/action.yml +++ b/.github/actions/config-variations/action.yml @@ -8,7 +8,7 @@ inputs: description: 'GitHub token' required: true tests: - description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized' + description: 'List of tests to run (space-separated IDs) or "all" for all tests. Available IDs: pct-enabled, pct-enabled-broken, signing-bound-exhausted, reduce-ram, reduce-ram-pct, custom-alloc-heap, custom-zeroize, native-cap-ON, native-cap-OFF, native-cap-ID_AA64PFR1_EL1, native-cap-CPUID_AVX2, no-asm, serial-fips202, x3-keccak, custom-randombytes, custom-memcpy, custom-memset, custom-stdlib, nblocks-1, nblocks-4, nblocks-6, keygen-only, sign-only, verify-only, keygen-sign, keygen-verify, sign-verify, no-randomized' required: false default: 'all' opt: @@ -236,6 +236,40 @@ runs: alloc: false # Requires custom config sign_hook: false # Requires custom config rng_fail: true + # On AArch64, build with SHA3 so that the native x3 hybrid kernel is used; + # elsewhere, this tests the C fallback of the 3-way batched Keccak path. + - name: "3-way batched Keccak (x3, AArch64 with SHA3)" + if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch == 'ARM64' }} + uses: ./.github/actions/functest + with: + gh_token: ${{ inputs.gh_token }} + platform: native + cflags: "${{ inputs.extra_cflags }} -std=c11 -march=armv8.4-a+sha3 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + func: true + kat: true + acvp: true + opt: ${{ inputs.opt }} + examples: false # Some examples use a custom config themselves + alloc: false # Requires custom config + sign_hook: false # Requires custom config + rng_fail: true + - name: "3-way batched Keccak (x3, C fallback)" + if: ${{ (inputs.tests == 'all' || contains(inputs.tests, 'x3-keccak')) && runner.arch != 'ARM64' }} + uses: ./.github/actions/functest + with: + gh_token: ${{ inputs.gh_token }} + platform: native + cflags: "${{ inputs.extra_cflags }} -std=c11 -D_GNU_SOURCE -Itest -DMLD_CONFIG_FILE=\\\"configs/x3_keccak_config.h\\\" -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + func: true + kat: true + acvp: true + opt: ${{ inputs.opt }} + examples: false # Some examples use a custom config themselves + alloc: false # Requires custom config + sign_hook: false # Requires custom config + rng_fail: true - name: "Custom randombytes" if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'custom-randombytes') }} uses: ./.github/actions/functest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbc9dc3347..50a5d03fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,7 +187,7 @@ jobs: strategy: fail-fast: false matrix: - backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar] + backend: [x1_scalar, x1_v84a, x2_v84a, x4_v8a_scalar, x4_v8a_v84a_scalar, x3_v84a_scalar] runs-on: macos-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -207,7 +207,8 @@ jobs: platform: native opt: 'opt' examples: 'false' - cflags: "-DMLDSA_DEBUG -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" + # The x3 backend is only used by the samplers with MLD_CONFIG_USE_X3. + cflags: "-DMLDSA_DEBUG ${{ matrix.backend == 'x3_v84a_scalar' && '-DMLD_CONFIG_USE_X3 ' || '' }}-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" ldflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" check_namespace: 'false' extra_args: "--fips202-aarch64-backend ${{ matrix.backend }}" @@ -549,7 +550,7 @@ jobs: - name: native-cap ids: native-cap-ON native-cap-OFF native-cap-ID_AA64PFR1_EL1 native-cap-CPUID_AVX2 - name: asm-fips202 - ids: no-asm serial-fips202 + ids: no-asm serial-fips202 x3-keccak - name: nblocks ids: nblocks-1 nblocks-4 nblocks-6 - name: reduced-api diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml index ce247bcaaa..645ebe93ea 100644 --- a/.github/workflows/ci_ec2_reusable.yml +++ b/.github/workflows/ci_ec2_reusable.yml @@ -240,6 +240,8 @@ jobs: tests all --no-examples --opt opt --cflags="-DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_scalar.h\\\"" make clean tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x4_v8a_v84a_scalar.h\\\"" + make clean + tests all --no-examples --opt opt --cflags="-march=armv8.4-a+sha3 -DMLD_CONFIG_USE_X3 -DMLD_CONFIG_FIPS202_BACKEND_FILE=\\\"fips202/native/aarch64/x3_v84a_scalar.h\\\"" stop-ec2-runner: name: Stop instance (${{ inputs.ec2_instance_type }}) permissions: diff --git a/scripts/tests b/scripts/tests index 1f6c5bb06e..88dd4b0674 100755 --- a/scripts/tests +++ b/scripts/tests @@ -1427,6 +1427,7 @@ def cli(): "x2_v84a", "x4_v8a_scalar", "x4_v8a_v84a_scalar", + "x3_v84a_scalar", ], default="auto", type=str, From b6223256dd7b7d98aa52ce08963d8272c026ded1 Mon Sep 17 00:00:00 2001 From: Soumya Bhattacharya Date: Wed, 21 Oct 2026 14:44:04 -0700 Subject: [PATCH 8/8] bench: Benchmark the x3 Keccak on Graviton 5 Use MLD_CONFIG_USE_X3 instead of MLD_SYS_AARCH64_FAST_SHA3, which would select x2_v84a ahead of x3. Signed-off-by: Soumya Bhattacharya --- .github/workflows/bench.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 89c27b247c..0e609f3472 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -167,7 +167,9 @@ jobs: - name: Graviton5 ec2_instance_type: c9g.medium ec2_ami: ubuntu (aarch64) - archflags: -march=armv9-a+sha3 -DMLD_SYS_AARCH64_FAST_SHA3 + # 3-way batched Keccak (x3). Not combined with MLD_SYS_AARCH64_FAST_SHA3, + # which selects the x2_v84a backend before MLD_CONFIG_USE_X3 is checked. + archflags: -march=armv9-a+sha3 -DMLD_CONFIG_USE_X3 cflags: "-flto -DMLD_FORCE_AARCH64" ldflags: "-flto" perf: PERF