Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn __indcpa_keypair(#[spill_to_mmx] reg u64 pkp skp, reg ptr u8[MLKEM_SYMBYTES]

for i=0 to MLKEM_SYMBYTES/8
{
#declassify t64 = buf[:u64 i];
#[declassify] t64 = buf[:u64 i];
publicseed[:u64 i] = t64;
t64 = buf[:u64 i + MLKEM_SYMBYTES/8];
noiseseed[:u64 i] = t64;
Expand Down Expand Up @@ -91,7 +91,7 @@ fn __indcpa_enc_0(#[mmx] reg u64 sctp, reg ptr u8[MLKEM_INDCPA_MSGBYTES] msgp, r
pkp += MLKEM_POLYVECBYTES;
while (i < MLKEM_SYMBYTES/8)
{
#declassify t64 = [:u64 pkp];
#[declassify] t64 = [:u64 pkp];
publicseed.[:u64 8 * (uint)i] = t64;
pkp += 8;
i += 1;
Expand Down Expand Up @@ -163,7 +163,7 @@ fn __indcpa_enc_1(
pkp += MLKEM_POLYVECBYTES;
while (i < MLKEM_SYMBYTES/8)
{
#declassify t64 = [:u64 pkp];
#[declassify] t64 = [:u64 pkp];
publicseed.[:u64 8*(uint)i] = t64;
pkp += 8;
i += 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ fn __indcpa_keypair(#[mmx] reg u64 spkp, #[mmx] reg u64 sskp, reg ptr u8[MLKEM_S

for i=0 to MLKEM_SYMBYTES/8
{
#declassify t64 = buf[:u64 i];
#[declassify] t64 = buf[:u64 i];
publicseed[:u64 i] = t64;
t64 = buf[:u64 i + MLKEM_SYMBYTES/8];
noiseseed[:u64 i] = t64;
Expand Down Expand Up @@ -105,7 +105,7 @@ fn __indcpa_enc(#[mmx] reg u64 sctp, reg ptr u8[32] msgp, reg u64 pkp, reg ptr u
pkp += MLKEM_POLYVECBYTES;
while (i < MLKEM_SYMBYTES/8)
{
#declassify t64 = [:u64 pkp];
#[declassify] t64 = [:u64 pkp];
publicseed.[:u64 8 * (uint)i] = t64;
pkp += 8;
i += 1;
Expand Down Expand Up @@ -181,7 +181,7 @@ fn __iindcpa_enc(reg ptr u8[MLKEM_CT_LEN] ctp, reg ptr u8[32] msgp, reg u64 pkp,
pkp += MLKEM_POLYVECBYTES;
while (i < MLKEM_SYMBYTES/8)
{
#declassify t64 = [:u64 pkp];
#[declassify] t64 = [:u64 pkp];
publicseed.[:u64 8*(uint)i] = t64;
pkp += 8;
i += 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,14 +247,14 @@ fn ntt_levels0t1(reg ptr u32[256] poly_ptr, reg u256 q, inline int offset)
poly4, poly6 = ntt_butterfly(poly4, poly6, zeta_qinv, zeta_qinv, zeta, zeta, q);
poly5, poly7 = ntt_butterfly(poly5, poly7, zeta_qinv, zeta_qinv, zeta, zeta, q);

#VMOVDQU_256(poly_ptr.[:u256 (32 * (0*4 + offset))]) = poly0;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (1*4 + offset))]) = poly1;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (2*4 + offset))]) = poly2;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (3*4 + offset))]) = poly3;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (4*4 + offset))]) = poly4;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (5*4 + offset))]) = poly5;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (6*4 + offset))]) = poly6;
#VMOVDQU_256(poly_ptr.[:u256 (32 * (7*4 + offset))]) = poly7;
poly_ptr.[:u256 (32 * (0*4 + offset))] = poly0;
poly_ptr.[:u256 (32 * (1*4 + offset))] = poly1;
poly_ptr.[:u256 (32 * (2*4 + offset))] = poly2;
poly_ptr.[:u256 (32 * (3*4 + offset))] = poly3;
poly_ptr.[:u256 (32 * (4*4 + offset))] = poly4;
poly_ptr.[:u256 (32 * (5*4 + offset))] = poly5;
poly_ptr.[:u256 (32 * (6*4 + offset))] = poly6;
poly_ptr.[:u256 (32 * (7*4 + offset))] = poly7;

return poly_ptr;
}
Expand Down Expand Up @@ -376,14 +376,14 @@ fn ntt_levels2t7(reg ptr u32[256] poly_ptr, reg u256 q, inline int offset)
zeta1 = #VMOVSHDUP_256(zeta0);
polyx, poly7 = ntt_butterfly(polyx, poly7, zeta_qinv0, zeta_qinv1, zeta0, zeta1, q);

#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 0)]) = poly5;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 1)]) = poly4;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 2)]) = poly3;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 3)]) = poly2;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 4)]) = poly1;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 5)]) = poly0;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 6)]) = polyx;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 7)]) = poly7;
poly_ptr.[:u256 32 * (8*offset + 0)] = poly5;
poly_ptr.[:u256 32 * (8*offset + 1)] = poly4;
poly_ptr.[:u256 32 * (8*offset + 2)] = poly3;
poly_ptr.[:u256 32 * (8*offset + 3)] = poly2;
poly_ptr.[:u256 32 * (8*offset + 4)] = poly1;
poly_ptr.[:u256 32 * (8*offset + 5)] = poly0;
poly_ptr.[:u256 32 * (8*offset + 6)] = polyx;
poly_ptr.[:u256 32 * (8*offset + 7)] = poly7;

return poly_ptr;
}
Expand Down Expand Up @@ -538,14 +538,14 @@ fn invntt_levels0t5(reg ptr u32[256] poly_ptr, reg u256 q, inline int offset) ->
poly2, poly4 = invntt_butterfly(poly2, poly4, zeta_qinv0, zeta_qinv0, zeta0, zeta0, q);
poly0, poly7 = invntt_butterfly(poly0, poly7, zeta_qinv0, zeta_qinv0, zeta0, zeta0, q);

#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 0)]) = poly5;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 1)]) = poly6;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 2)]) = poly2;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 3)]) = poly0;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 4)]) = polyx;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 5)]) = poly1;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 6)]) = poly4;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8*offset + 7)]) = poly7;
poly_ptr.[:u256 32 * (8*offset + 0)] = poly5;
poly_ptr.[:u256 32 * (8*offset + 1)] = poly6;
poly_ptr.[:u256 32 * (8*offset + 2)] = poly2;
poly_ptr.[:u256 32 * (8*offset + 3)] = poly0;
poly_ptr.[:u256 32 * (8*offset + 4)] = polyx;
poly_ptr.[:u256 32 * (8*offset + 5)] = poly1;
poly_ptr.[:u256 32 * (8*offset + 6)] = poly4;
poly_ptr.[:u256 32 * (8*offset + 7)] = poly7;

return poly_ptr;
}
Expand Down Expand Up @@ -585,10 +585,10 @@ fn invntt_levels6t7(reg ptr u32[256] poly_ptr, reg u256 q, inline int offset) ->
poly2, poly6 = invntt_butterfly(poly2, poly6, zeta_qinv, zeta_qinv, zeta, zeta, q);
poly3, poly7 = invntt_butterfly(poly3, poly7, zeta_qinv, zeta_qinv, zeta, zeta, q);

#VMOVDQU_256(poly_ptr.[:u256 32 * (16 + offset)]) = poly4;
#VMOVDQU_256(poly_ptr.[:u256 32 * (20 + offset)]) = poly5;
#VMOVDQU_256(poly_ptr.[:u256 32 * (24 + offset)]) = poly6;
#VMOVDQU_256(poly_ptr.[:u256 32 * (28 + offset)]) = poly7;
poly_ptr.[:u256 32 * (16 + offset)] = poly4;
poly_ptr.[:u256 32 * (20 + offset)] = poly5;
poly_ptr.[:u256 32 * (24 + offset)] = poly6;
poly_ptr.[:u256 32 * (28 + offset)] = poly7;

// Divide by 256 and map to Montgomery domain
// TODO: Understand why we only have to do this with half of the
Expand Down Expand Up @@ -641,10 +641,10 @@ fn invntt_levels6t7(reg ptr u32[256] poly_ptr, reg u256 q, inline int offset) ->
poly2 = #VPBLEND_8u32(poly2, poly4, 0xAA);
poly3 = #VPBLEND_8u32(poly3, poly5, 0xAA);

#VMOVDQU_256(poly_ptr.[:u256 32 * (0 + offset)]) = poly0;
#VMOVDQU_256(poly_ptr.[:u256 32 * (4 + offset)]) = poly1;
#VMOVDQU_256(poly_ptr.[:u256 32 * (8 + offset)]) = poly2;
#VMOVDQU_256(poly_ptr.[:u256 32 * (12 + offset)]) = poly3;
poly_ptr.[:u256 32 * (0 + offset)] = poly0;
poly_ptr.[:u256 32 * (4 + offset)] = poly1;
poly_ptr.[:u256 32 * (8 + offset)] = poly2;
poly_ptr.[:u256 32 * (12 + offset)] = poly3;

return poly_ptr;
}
Expand Down Expand Up @@ -694,14 +694,14 @@ fn ntt_transpose_inner_avx2(reg ptr u32[256] poly_ptr, reg u64 offset) -> reg pt
ymm5, ymm4 = shuffle2(ymm3, ymm4);
ymm3, ymm11 = shuffle2(ymm10, ymm11);

#VMOVDQU_256(poly_ptr.[:u256 32 * 0 + (uint) offset]) = ymm9;
#VMOVDQU_256(poly_ptr.[:u256 32 * 1 + (uint) offset]) = ymm8;
#VMOVDQU_256(poly_ptr.[:u256 32 * 2 + (uint) offset]) = ymm7;
#VMOVDQU_256(poly_ptr.[:u256 32 * 3 + (uint) offset]) = ymm6;
#VMOVDQU_256(poly_ptr.[:u256 32 * 4 + (uint) offset]) = ymm5;
#VMOVDQU_256(poly_ptr.[:u256 32 * 5 + (uint) offset]) = ymm4;
#VMOVDQU_256(poly_ptr.[:u256 32 * 6 + (uint) offset]) = ymm3;
#VMOVDQU_256(poly_ptr.[:u256 32 * 7 + (uint) offset]) = ymm11;
poly_ptr.[:u256 32 * 0 + (uint) offset] = ymm9;
poly_ptr.[:u256 32 * 1 + (uint) offset] = ymm8;
poly_ptr.[:u256 32 * 2 + (uint) offset] = ymm7;
poly_ptr.[:u256 32 * 3 + (uint) offset] = ymm6;
poly_ptr.[:u256 32 * 4 + (uint) offset] = ymm5;
poly_ptr.[:u256 32 * 5 + (uint) offset] = ymm4;
poly_ptr.[:u256 32 * 6 + (uint) offset] = ymm3;
poly_ptr.[:u256 32 * 7 + (uint) offset] = ymm11;

return poly_ptr;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ fn poly_add(reg ptr u32[Li2_polydeg] f g result)
while (offset < 4 * Li2_polydeg) {
v256 = #VMOVDQU_256(f.[:u256 offset]);
v256 = #VPADD_8u32(v256, g.[:u256 offset]);
#VMOVDQU_256(result.[:u256 offset]) = v256;
result.[:u256 offset] = v256;

offset += 32;
}
Expand All @@ -39,7 +39,7 @@ fn poly_subtract(reg ptr u32[Li2_polydeg] f g result)
while (offset < 4 * Li2_polydeg) {
v256 = #VMOVDQU_256(f.[:u256 offset]);
v256 = #VPSUB_8u32(v256, g.[:u256 offset]);
#VMOVDQU_256(result.[:u256 offset]) = v256;
result.[:u256 offset] = v256;

offset += 32;
}
Expand All @@ -56,7 +56,7 @@ fn poly_accumulate(reg ptr u32[Li2_polydeg] f acc)
while (offset < 4 * Li2_polydeg) {
v256 = #VMOVDQU_256(acc.[:u256 offset]);
v256 = #VPADD_8u32(v256, f.[:u256 offset]);
#VMOVDQU_256(acc.[:u256 offset]) = v256;
acc.[:u256 offset] = v256;

offset += 32;
}
Expand All @@ -83,7 +83,7 @@ fn poly_reduce32(reg ptr u32[Li2_polydeg] poly)
// t = a - t*Q
t = #VPMULL_8u32(t, q);
a = #VPSUB_8u32(a, t);
#VMOVDQU_256(poly.[:u256 offset]) = a;
poly.[:u256 offset] = a;

offset += 32;
}
Expand Down Expand Up @@ -130,7 +130,7 @@ fn poly_pointwise_montgomery(reg ptr u32[Li2_polydeg] fft_f fft_g fft_prod)

prod_lo = #VMOVSHDUP_256(prod_lo);
prod_lo = #VPBLEND_8u32(prod_lo, prod_hi, 0xAA);
#VMOVDQU_256(fft_prod.[:u256 offset]) = prod_lo;
fft_prod.[:u256 offset] = prod_lo;

offset += 32;
}
Expand All @@ -147,7 +147,7 @@ fn zero_poly(reg ptr u32[Li2_polydeg] f)
?{}, offset = #set0_64();

while (offset < 4 * Li2_polydeg) {
#VMOVDQU_256(f.[:u256 offset]) = zero;
f.[:u256 offset] = zero;
offset += 32;
}
return f;
Expand All @@ -168,7 +168,7 @@ fn poly_caddq(reg ptr u32[Li2_polydeg] poly)
mask = #VPCMPGT_8u32(zero, v256);
add = #VPAND_256(q, mask);
v256 = #VPADD_8u32(v256, add);
#VMOVDQU_256(poly.[:u256 offset]) = v256;
poly.[:u256 offset] = v256;

offset += 32;
}
Expand Down
Loading