diff --git a/wolfssl-sys/build.rs b/wolfssl-sys/build.rs index 87845d9..0da5bd8 100644 --- a/wolfssl-sys/build.rs +++ b/wolfssl-sys/build.rs @@ -44,6 +44,7 @@ fn copy_wolfssl(dest: &Path) -> std::io::Result { const PATCH_DIR: &str = "patches"; const PATCHES: &[&str] = &[ + "revert-aarch64-poly1305-asm-improve-performance.patch", "include-private-key-fields-for-kyber.patch", "make-kyber-mlkem-available.patch", "fix-kyber-mlkem-benchmark.patch", diff --git a/wolfssl-sys/patches/revert-aarch64-poly1305-asm-improve-performance.patch b/wolfssl-sys/patches/revert-aarch64-poly1305-asm-improve-performance.patch new file mode 100644 index 0000000..e0045ad --- /dev/null +++ b/wolfssl-sys/patches/revert-aarch64-poly1305-asm-improve-performance.patch @@ -0,0 +1,688 @@ +From f8b56c4a3230238760f4865d5112a4c7f727f49c Mon Sep 17 00:00:00 2001 +From: Thomas Yau +Date: Tue, 3 Dec 2024 12:32:09 +0800 +Subject: [PATCH] Revert Aarch64 Poly1305 ASM: Improve performance PR + +Revert Aarch64 Poly1305 ASM: Improve performance +- 3725594020ce965de056893119ced82995bf6d01 +--- + wolfcrypt/src/port/arm/armv8-poly1305.c | 589 +++++++++++++----------- + wolfssl/wolfcrypt/poly1305.h | 1 - + 2 files changed, 316 insertions(+), 274 deletions(-) + +diff --git a/wolfcrypt/src/port/arm/armv8-poly1305.c b/wolfcrypt/src/port/arm/armv8-poly1305.c +index a258f3607..3413ce171 100644 +--- a/wolfcrypt/src/port/arm/armv8-poly1305.c ++++ b/wolfcrypt/src/port/arm/armv8-poly1305.c +@@ -54,112 +54,137 @@ static WC_INLINE void poly1305_blocks_aarch64_16(Poly1305* ctx, + const unsigned char *m, size_t bytes) + { + __asm__ __volatile__ ( +- /* Check for zero bytes to do. */ + "CMP %[bytes], %[POLY1305_BLOCK_SIZE] \n\t" +- "BLO L_poly1305_aarch64_16_done_%= \n\t" +- +- "MOV x12, #1 \n\t" +- /* Load h */ +- "LDP w4, w5, [%[ctx_h], #0] \n\t" +- "LDP w6, w7, [%[ctx_h], #8] \n\t" +- "LDR w8, [%[ctx_h], #16] \n\t" +- /* Base 26 -> Base 64 */ +- "ORR x4, x4, x5, LSL #26\n\t" +- "ORR x4, x4, x6, LSL #52\n\t" +- "LSR x5, x6, #12\n\t" +- "ORR x5, x5, x7, LSL #14\n\t" +- "ORR x5, x5, x8, LSL #40\n\t" +- "LSR x6, x8, #24\n\t" +- /* Load r */ +- "LDP x8, x9, %[ctx_r64] \n\t" +- "SUB %[finished], x12, %[finished]\n\t" ++ "BLO L_poly1305_aarch64_16_64_done_%= \n\t" ++ /* Load r and h */ ++ "LDP x21, x23, %[ctx_r] \n\t" ++ "LDR w25, %[ctx_r_4] \n\t" ++ "LDP x2, x4, %[ctx_h] \n\t" ++ "LDR w6, %[ctx_h_4] \n\t" ++ "LSR x22, x21, #32 \n\t" ++ "LSR x24, x23, #32 \n\t" ++ "LSR x3, x2, #32 \n\t" ++ "LSR x5, x4, #32 \n\t" ++ "AND x21, x21, #0x3ffffff \n\t" ++ "AND x23, x23, #0x3ffffff \n\t" ++ "AND x2, x2, #0x3ffffff \n\t" ++ "AND x4, x4, #0x3ffffff \n\t" ++ /* s1 = r1 * 5; */ ++ /* s2 = r2 * 5; */ ++ /* s3 = r3 * 5; */ ++ /* s4 = r4 * 5; */ ++ "MOV x15, #5 \n\t" ++ "CMP %[finished], #0 \n\t" ++ "MUL w7, w22, w15 \n\t" ++ "CSET %[finished], EQ \n\t" ++ "MUL w8, w23, w15 \n\t" ++ "LSL %[finished], %[finished], #24 \n\t" ++ "MUL w9, w24, w15 \n\t" ++ "MOV x14, #0x3ffffff \n\t" ++ "MUL w10, w25, w15 \n\t" + "\n" + ".align 2 \n\t" +- "L_poly1305_aarch64_16_loop_%=: \n\t" +- /* Load m */ +- "LDR x10, [%[m]] \n\t" +- "LDR x11, [%[m], 8] \n\t" +- /* Add m and !finished at bit 128. */ +- "ADDS x4, x4, x10 \n\t" +- "ADCS x5, x5, x11 \n\t" +- "ADC x6, x6, %[finished] \n\t" +- +- /* r * h */ +- /* r0 * h0 */ +- "MUL x12, x8, x4\n\t" +- "UMULH x13, x8, x4\n\t" +- /* r0 * h1 */ +- "MUL x16, x8, x5\n\t" +- "UMULH x14, x8, x5\n\t" +- /* r1 * h0 */ +- "MUL x15, x9, x4\n\t" +- "ADDS x13, x13, x16\n\t" +- "UMULH x17, x9, x4\n\t" +- "ADC x14, x14, xzr\n\t" +- "ADDS x13, x13, x15\n\t" +- /* r0 * h2 */ +- "MUL x16, x8, x6\n\t" +- "ADCS x14, x14, x17\n\t" +- "UMULH x17, x8, x6\n\t" +- "ADC x15, xzr, xzr\n\t" +- "ADDS x14, x14, x16\n\t" +- /* r1 * h1 */ +- "MUL x16, x9, x5\n\t" +- "ADC x15, x15, x17\n\t" +- "UMULH x19, x9, x5\n\t" +- "ADDS x14, x14, x16\n\t" +- /* r1 * h2 */ +- "MUL x17, x9, x6\n\t" +- "ADCS x15, x15, x19\n\t" +- "UMULH x19, x9, x6\n\t" +- "ADC x16, xzr, xzr\n\t" +- "ADDS x15, x15, x17\n\t" +- "ADC x16, x16, x19\n\t" +- /* h' = x12, x13, x14, x15, x16 */ +- +- /* h' mod 2^130 - 5 */ +- /* Get top two bits from h[2]. */ +- "AND x6, x14, 3\n\t" +- /* Get high bits from h[2]. */ +- "AND x14, x14, -4\n\t" +- /* Add top bits * 4. */ +- "ADDS x4, x12, x14\n\t" +- "ADCS x5, x13, x15\n\t" +- "ADC x6, x6, x16\n\t" +- /* Move down 2 bits. */ +- "EXTR x14, x15, x14, 2\n\t" +- "EXTR x15, x16, x15, 2\n\t" +- /* Add top bits. */ +- "ADDS x4, x4, x14\n\t" +- "ADCS x5, x5, x15\n\t" +- "ADC x6, x6, xzr\n\t" +- +- "SUBS %[bytes], %[bytes], %[POLY1305_BLOCK_SIZE]\n\t" +- "ADD %[m], %[m], %[POLY1305_BLOCK_SIZE]\n\t" +- "BGT L_poly1305_aarch64_16_loop_%=\n\t" +- +- /* Base 64 -> Base 26 */ +- "MOV x10, #0x3ffffff\n\t" +- "EXTR x8, x6, x5, #40\n\t" +- "AND x7, x10, x5, LSR #14\n\t" +- "EXTR x6, x5, x4, #52\n\t" +- "AND x5, x10, x4, LSR #26\n\t" +- "AND x4, x4, x10\n\t" +- "AND x6, x6, x10\n\t" +- "AND x8, x8, x10\n\t" +- "STP w4, w5, [%[ctx_h], #0] \n\t" +- "STP w6, w7, [%[ctx_h], #8] \n\t" +- "STR w8, [%[ctx_h], #16] \n\t" ++ "L_poly1305_aarch64_16_64_loop_%=: \n\t" ++ /* t0 = U8TO64(&m[0]); */ ++ /* t1 = U8TO64(&m[8]); */ ++ "LDP x16, x17, [%[m]], #16 \n\t" ++ /* h0 += (U8TO32(m + 0)) & 0x3ffffff; */ ++ "AND x26, x16, #0x3ffffff \n\t" ++ "ADD x2, x2, x26 \n\t" ++ /* h1 += (U8TO32(m + 3) >> 2) & 0x3ffffff; */ ++ "AND x26, x14, x16, LSR #26 \n\t" ++ "ADD x3, x3, x26 \n\t" ++ /* h2 += (U8TO32(m + 6) >> 4) & 0x3ffffff; */ ++ "EXTR x26, x17, x16, #52 \n\t" ++ "AND x26, x26, #0x3ffffff \n\t" ++ "ADD x4, x4, x26 \n\t" ++ /* h3 += (U8TO32(m + 9) >> 6) & 0x3ffffff; */ ++ "AND x26, x14, x17, LSR #14 \n\t" ++ "ADD x5, x5, x26 \n\t" ++ /* h4 += (U8TO32(m + 12) >> 8) | hibit; */ ++ "ORR x17, %[finished], x17, LSR #40 \n\t" ++ "ADD x6, x6, x17 \n\t" ++ /* d0 = h0 * r0 + h1 * s4 + h2 * s3 + h3 * s2 + h4 * s1 */ ++ /* d1 = h0 * r1 + h1 * r0 + h2 * s4 + h3 * s3 + h4 * s2 */ ++ /* d2 = h0 * r2 + h1 * r1 + h2 * r0 + h3 * s4 + h4 * s3 */ ++ /* d3 = h0 * r3 + h1 * r2 + h2 * r1 + h3 * r0 + h4 * s4 */ ++ /* d4 = h0 * r4 + h1 * r3 + h2 * r2 + h3 * r1 + h4 * r0 */ ++ "MUL x16, x2, x21 \n\t" ++ "MUL x17, x2, x22 \n\t" ++ "MUL x26, x2, x23 \n\t" ++ "MUL x19, x2, x24 \n\t" ++ "MUL x20, x2, x25 \n\t" ++ "MADD x16, x3, x10, x16 \n\t" ++ "MADD x17, x3, x21, x17 \n\t" ++ "MADD x26, x3, x22, x26 \n\t" ++ "MADD x19, x3, x23, x19 \n\t" ++ "MADD x20, x3, x24, x20 \n\t" ++ "MADD x16, x4, x9, x16 \n\t" ++ "MADD x17, x4, x10, x17 \n\t" ++ "MADD x26, x4, x21, x26 \n\t" ++ "MADD x19, x4, x22, x19 \n\t" ++ "MADD x20, x4, x23, x20 \n\t" ++ "MADD x16, x5, x8, x16 \n\t" ++ "MADD x17, x5, x9, x17 \n\t" ++ "MADD x26, x5, x10, x26 \n\t" ++ "MADD x19, x5, x21, x19 \n\t" ++ "MADD x20, x5, x22, x20 \n\t" ++ "MADD x16, x6, x7, x16 \n\t" ++ "MADD x17, x6, x8, x17 \n\t" ++ "MADD x26, x6, x9, x26 \n\t" ++ "MADD x19, x6, x10, x19 \n\t" ++ "MADD x20, x6, x21, x20 \n\t" ++ /* d1 = d1 + d0 >> 26 */ ++ /* d2 = d2 + d1 >> 26 */ ++ /* d3 = d3 + d2 >> 26 */ ++ /* d4 = d4 + d3 >> 26 */ ++ /* h0 = d0 & 0x3ffffff */ ++ /* h1 = d1 & 0x3ffffff */ ++ /* h2 = d2 & 0x3ffffff */ ++ /* h0 = h0 + (d4 >> 26) * 5 */ ++ /* h1 = h1 + h0 >> 26 */ ++ /* h3 = d3 & 0x3ffffff */ ++ /* h4 = d4 & 0x3ffffff */ ++ /* h0 = h0 & 0x3ffffff */ ++ "ADD x17, x17, x16, LSR #26 \n\t" ++ "ADD x20, x20, x19, LSR #26 \n\t" ++ "AND x16, x16, #0x3ffffff \n\t" ++ "LSR x2, x20, #26 \n\t" ++ "AND x19, x19, #0x3ffffff \n\t" ++ "MADD x16, x2, x15, x16 \n\t" ++ "ADD x26, x26, x17, LSR #26 \n\t" ++ "AND x17, x17, #0x3ffffff \n\t" ++ "AND x20, x20, #0x3ffffff \n\t" ++ "ADD x19, x19, x26, LSR #26 \n\t" ++ "AND x4, x26, #0x3ffffff \n\t" ++ "ADD x3, x17, x16, LSR #26 \n\t" ++ "AND x2, x16, #0x3ffffff \n\t" ++ "ADD x6, x20, x19, LSR #26 \n\t" ++ "AND x5, x19, #0x3ffffff \n\t" ++ "SUB %[bytes], %[bytes], %[POLY1305_BLOCK_SIZE] \n\t" ++ "CMP %[bytes], %[POLY1305_BLOCK_SIZE] \n\t" ++ "BHS L_poly1305_aarch64_16_64_loop_%= \n\t" ++ /* Store h */ ++ "ORR x2, x2, x3, LSL #32 \n\t" ++ "ORR x4, x4, x5, LSL #32 \n\t" ++ "STP x2, x4, %[ctx_h] \n\t" ++ "STR w6, %[ctx_h_4] \n\t" + "\n" + ".align 2 \n\t" +- "L_poly1305_aarch64_16_done_%=: \n\t" +- : [bytes] "+r" (bytes), [m] "+r" (m) ++ "L_poly1305_aarch64_16_64_done_%=: \n\t" ++ : [ctx_h] "+m" (ctx->h[0]), ++ [ctx_h_4] "+m" (ctx->h[4]), ++ [bytes] "+r" (bytes), ++ [m] "+r" (m) + : [POLY1305_BLOCK_SIZE] "I" (POLY1305_BLOCK_SIZE), +- [ctx_r64] "m" (ctx->r64[0]), [ctx_h] "r" (ctx->h), ++ [ctx_r] "m" (ctx->r[0]), ++ [ctx_r_4] "m" (ctx->r[4]), + [finished] "r" ((word64)ctx->finished) + : "memory", "cc", +- "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", +- "x15", "x16", "x17", "x19" ++ "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9", "w10", "w15", ++ "w21", "w22", "w23", "w24", "w25", "x2", "x3", "x4", "x5", "x6", ++ "x7", "x8", "x9", "x10", "x14", "x15", "x16", "x17", "x19", "x20", ++ "x21", "x22", "x23", "x24", "x25", "x26" + ); + } + +@@ -886,147 +911,151 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz) + "LDP x10, x11, [%[key], #16] \n\t" + /* Load clamp */ + "LDP x12, x13, [%[clamp]] \n\t" +- /* Save pad for later */ +- "STP x10, x11, [%[ctx_pad]] \n\t" + /* Apply clamp */ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + "AND x8, x8, x12 \n\t" + "AND x9, x9, x13 \n\t" +- "STP x8, x9, [%[ctx_r64]] \n\t" +- /* 128-bits: Base 64 -> Base 26 */ +- "MOV x20, #0x3ffffff\n\t" +- "LSR x15, x9, #40\n\t" +- "AND x14, x20, x9, LSR #14\n\t" +- "EXTR x13, x9, x8, #52\n\t" +- "AND x12, x20, x8, LSR #26\n\t" +- "AND x11, x8, x20\n\t" +- "AND x13, x13, x20\n\t" +- "AND x15, x15, x20\n\t" +- "STP w11, w12, [%[ctx_r], #0] \n\t" +- "STP w13, w14, [%[ctx_r], #8] \n\t" +- "STR w15, [%[ctx_r], #16] \n\t" +- ++ "MOV x19, xzr \n\t" ++ "MOV x20, xzr \n\t" ++ "MOV x21, xzr \n\t" ++ "MOV x22, xzr \n\t" ++ "MOV x23, xzr \n\t" ++ "BFI x19, x8, #0, #26 \n\t" ++ "LSR x8, x8, #26 \n\t" ++ "BFI x20, x8, #0, #26 \n\t" ++ "LSR x8, x8, #26 \n\t" ++ "BFI x21, x8, #0, #12 \n\t" ++ "BFI x21, x9, #12, #14 \n\t" ++ "LSR x9, x9, #14 \n\t" ++ "BFI x22, x9, #0, #26 \n\t" ++ "LSR x9, x9, #26 \n\t" ++ "BFI x23, x9, #0, #24 \n\t" + /* Compute r^2 */ +- /* r0 * r0 */ +- "MUL x12, x8, x8\n\t" +- "UMULH x13, x8, x8\n\t" +- /* 2 * r0 * r1 */ +- "MUL x15, x8, x9\n\t" +- "UMULH x16, x8, x9\n\t" +- "ADDS x13, x13, x15\n\t" +- "ADC x14, xzr, x16\n\t" +- "ADDS x13, x13, x15\n\t" +- "ADCS x14, x14, x16\n\t" +- "ADC x15, xzr, xzr\n\t" +- /* r1 * r1 */ +- "MUL x16, x9, x9\n\t" +- "UMULH x17, x9, x9\n\t" +- "ADDS x14, x14, x16\n\t" +- "ADC x15, x15, x17\n\t" ++ /* r*5 */ ++ "MOV x8, #5 \n\t" ++ "MUL x24, x20, x8 \n\t" ++ "MUL x25, x21, x8 \n\t" ++ "MUL x26, x22, x8 \n\t" ++ "MUL x27, x23, x8 \n\t" ++ /* d = r*r */ ++ /* d0 = h0 * r0 + h1 * s4 + h2 * s3 + h3 * s2 + h4 * s1 */ ++ /* d1 = h0 * r1 + h1 * r0 + h2 * s4 + h3 * s3 + h4 * s2 */ ++ /* d2 = h0 * r2 + h1 * r1 + h2 * r0 + h3 * s4 + h4 * s3 */ ++ /* d3 = h0 * r3 + h1 * r2 + h2 * r1 + h3 * r0 + h4 * s4 */ ++ /* d4 = h0 * r4 + h1 * r3 + h2 * r2 + h3 * r1 + h4 * r0 */ ++ "MUL x14, x19, x19 \n\t" ++ "MUL x15, x19, x20 \n\t" ++ "MUL x16, x19, x21 \n\t" ++ "MUL x17, x19, x22 \n\t" ++ "MUL x7, x19, x23 \n\t" ++ "MADD x14, x20, x27, x14 \n\t" ++ "MADD x15, x20, x19, x15 \n\t" ++ "MADD x16, x20, x20, x16 \n\t" ++ "MADD x17, x20, x21, x17 \n\t" ++ "MADD x7, x20, x22, x7 \n\t" ++ "MADD x14, x21, x26, x14 \n\t" ++ "MADD x15, x21, x27, x15 \n\t" ++ "MADD x16, x21, x19, x16 \n\t" ++ "MADD x17, x21, x20, x17 \n\t" ++ "MADD x7, x21, x21, x7 \n\t" ++ "MADD x14, x22, x25, x14 \n\t" ++ "MADD x15, x22, x26, x15 \n\t" ++ "MADD x16, x22, x27, x16 \n\t" ++ "MADD x17, x22, x19, x17 \n\t" ++ "MADD x7, x22, x20, x7 \n\t" ++ "MADD x14, x23, x24, x14 \n\t" ++ "MADD x15, x23, x25, x15 \n\t" ++ "MADD x16, x23, x26, x16 \n\t" ++ "MADD x17, x23, x27, x17 \n\t" ++ "MADD x7, x23, x19, x7 \n\t" + /* r_2 = r^2 % P */ +- /* Get top two bits from r^2[2]. */ +- "AND x10, x14, 3\n\t" +- /* Get high bits from r^2[2]. */ +- "AND x14, x14, -4\n\t" +- /* Add top bits * 4. */ +- "ADDS x8, x12, x14\n\t" +- "ADCS x9, x13, x15\n\t" +- "ADC x10, x10, xzr\n\t" +- /* Move down 2 bits. */ +- "EXTR x14, x15, x14, 2\n\t" +- "LSR x15, x15, 2\n\t" +- /* Add top bits. */ +- "ADDS x8, x8, x14\n\t" +- "ADCS x9, x9, x15\n\t" +- "ADC x10, x10, xzr\n\t" +- /* 130-bits: Base 64 -> Base 26 */ +- "EXTR x15, x10, x9, #40\n\t" +- "AND x14, x20, x9, LSR #14\n\t" +- "EXTR x13, x9, x8, #52\n\t" +- "AND x12, x20, x8, LSR #26\n\t" +- "AND x11, x8, x20\n\t" +- "AND x13, x13, x20\n\t" +- "AND x15, x15, x20\n\t" +- /* Store r^2 */ +- "STP w11, w12, [%[ctx_r_2], #0] \n\t" +- "STP w13, w14, [%[ctx_r_2], #8] \n\t" +- "STR w15, [%[ctx_r_2], #16] \n\t" +- ++ "ADD x15, x15, x14, LSR #26 \n\t" ++ "ADD x7, x7, x17, LSR #26 \n\t" ++ "AND x14, x14, #0x3ffffff \n\t" ++ "LSR x9, x7, #26 \n\t" ++ "AND x17, x17, #0x3ffffff \n\t" ++ "MADD x14, x9, x8, x14 \n\t" ++ "ADD x16, x16, x15, LSR #26 \n\t" ++ "AND x15, x15, #0x3ffffff \n\t" ++ "AND x7, x7, #0x3ffffff \n\t" ++ "ADD x17, x17, x16, LSR #26 \n\t" ++ "AND x16, x16, #0x3ffffff \n\t" ++ "ADD x15, x15, x14, LSR #26 \n\t" ++ "AND x14, x14, #0x3ffffff \n\t" ++ "ADD x7, x7, x17, LSR #26 \n\t" ++ "AND x17, x17, #0x3ffffff \n\t" ++ /* Store r */ ++ "ORR x19, x19, x20, LSL #32 \n\t" ++ "ORR x21, x21, x22, LSL #32 \n\t" ++ "STP x19, x21, [%[ctx_r]] \n\t" ++ "STR w23, [%[ctx_r], #16] \n\t" ++ "MOV x8, #5 \n\t" ++ "MUL x24, x15, x8 \n\t" ++ "MUL x25, x16, x8 \n\t" ++ "MUL x26, x17, x8 \n\t" ++ "MUL x27, x7, x8 \n\t" + /* Compute r^4 */ +- /* r0 * r0 */ +- "MUL x12, x8, x8\n\t" +- "UMULH x13, x8, x8\n\t" +- /* 2 * r0 * r1 */ +- "MUL x15, x8, x9\n\t" +- "UMULH x16, x8, x9\n\t" +- "ADDS x13, x13, x15\n\t" +- "ADC x14, xzr, x16\n\t" +- "ADDS x13, x13, x15\n\t" +- "ADCS x14, x14, x16\n\t" +- "ADC x15, xzr, xzr\n\t" +- /* 2 * r0 * r2 */ +- "MUL x16, x8, x10\n\t" +- "UMULH x17, x8, x10\n\t" +- "ADDS x14, x14, x16\n\t" +- "ADC x15, x15, x17\n\t" +- "ADDS x14, x14, x16\n\t" +- "ADC x15, x15, x17\n\t" +- /* r1 * r1 */ +- "MUL x16, x9, x9\n\t" +- "UMULH x17, x9, x9\n\t" +- "ADDS x14, x14, x16\n\t" +- "ADCS x15, x15, x17\n\t" +- "ADC x16, xzr, xzr\n\t" +- /* 2 * r1 * r2 */ +- "MUL x17, x9, x10\n\t" +- "UMULH x19, x9, x10\n\t" +- "ADDS x15, x15, x17\n\t" +- "ADC x16, x16, x19\n\t" +- "ADDS x15, x15, x17\n\t" +- "ADC x16, x16, x19\n\t" +- /* r2 * r2 */ +- "MUL x17, x10, x10\n\t" +- "ADD x16, x16, x17\n\t" +- /* r_4 = r^4 % P */ +- /* Get top two bits from r^4[2]. */ +- "AND x10, x14, 3\n\t" +- /* Get high bits from r^4[2]. */ +- "AND x14, x14, -4\n\t" +- /* Add top bits * 4. */ +- "ADDS x8, x12, x14\n\t" +- "ADCS x9, x13, x15\n\t" +- "ADC x10, x10, x16\n\t" +- /* Move down 2 bits. */ +- "EXTR x14, x15, x14, 2\n\t" +- "EXTR x15, x16, x15, 2\n\t" +- "LSR x16, x16, 2\n\t" +- /* Add top bits. */ +- "ADDS x8, x8, x14\n\t" +- "ADCS x9, x9, x15\n\t" +- "ADC x10, x10, x16\n\t" +- /* Top again as it was 260 bits mod less than 130 bits. */ +- "AND x11, x10, -4\n\t" +- "AND x10, x10, 3\n\t" +- "ADD x11, x11, x11, LSR #2\n\t" +- "ADDS x8, x8, x11\n\t" +- "ADCS x9, x9, xzr\n\t" +- "ADC x10, x10, xzr\n\t" +- /* 130-bits: Base 64 -> Base 26 */ +- "EXTR x15, x10, x9, #40\n\t" +- "AND x14, x20, x9, LSR #14\n\t" +- "EXTR x13, x9, x8, #52\n\t" +- "AND x12, x20, x8, LSR #26\n\t" +- "AND x11, x8, x20\n\t" +- "AND x13, x13, x20\n\t" +- "AND x15, x15, x20\n\t" ++ /* d0 = h0 * r0 + h1 * s4 + h2 * s3 + h3 * s2 + h4 * s1 */ ++ /* d1 = h0 * r1 + h1 * r0 + h2 * s4 + h3 * s3 + h4 * s2 */ ++ /* d2 = h0 * r2 + h1 * r1 + h2 * r0 + h3 * s4 + h4 * s3 */ ++ /* d3 = h0 * r3 + h1 * r2 + h2 * r1 + h3 * r0 + h4 * s4 */ ++ /* d4 = h0 * r4 + h1 * r3 + h2 * r2 + h3 * r1 + h4 * r0 */ ++ "MUL x19, x14, x14 \n\t" ++ "MUL x20, x14, x15 \n\t" ++ "MUL x21, x14, x16 \n\t" ++ "MUL x22, x14, x17 \n\t" ++ "MUL x23, x14, x7 \n\t" ++ "MADD x19, x15, x27, x19 \n\t" ++ "MADD x20, x15, x14, x20 \n\t" ++ "MADD x21, x15, x15, x21 \n\t" ++ "MADD x22, x15, x16, x22 \n\t" ++ "MADD x23, x15, x17, x23 \n\t" ++ "MADD x19, x16, x26, x19 \n\t" ++ "MADD x20, x16, x27, x20 \n\t" ++ "MADD x21, x16, x14, x21 \n\t" ++ "MADD x22, x16, x15, x22 \n\t" ++ "MADD x23, x16, x16, x23 \n\t" ++ "MADD x19, x17, x25, x19 \n\t" ++ "MADD x20, x17, x26, x20 \n\t" ++ "MADD x21, x17, x27, x21 \n\t" ++ "MADD x22, x17, x14, x22 \n\t" ++ "MADD x23, x17, x15, x23 \n\t" ++ "MADD x19, x7, x24, x19 \n\t" ++ "MADD x20, x7, x25, x20 \n\t" ++ "MADD x21, x7, x26, x21 \n\t" ++ "MADD x22, x7, x27, x22 \n\t" ++ "MADD x23, x7, x14, x23 \n\t" ++ /* r^4 % P */ ++ "ADD x20, x20, x19, LSR #26 \n\t" ++ "ADD x23, x23, x22, LSR #26 \n\t" ++ "AND x19, x19, #0x3ffffff \n\t" ++ "LSR x9, x23, #26 \n\t" ++ "AND x22, x22, #0x3ffffff \n\t" ++ "MADD x19, x9, x8, x19 \n\t" ++ "ADD x21, x21, x20, LSR #26 \n\t" ++ "AND x20, x20, #0x3ffffff \n\t" ++ "AND x23, x23, #0x3ffffff \n\t" ++ "ADD x22, x22, x21, LSR #26 \n\t" ++ "AND x21, x21, #0x3ffffff \n\t" ++ "ADD x20, x20, x19, LSR #26 \n\t" ++ "AND x19, x19, #0x3ffffff \n\t" ++ "ADD x23, x23, x22, LSR #26 \n\t" ++ "AND x22, x22, #0x3ffffff \n\t" ++ /* Store r^2 */ ++ "ORR x14, x14, x15, LSL #32 \n\t" ++ "ORR x16, x16, x17, LSL #32 \n\t" ++ "STP x14, x16, [%[ctx_r_2]] \n\t" ++ "STR w7, [%[ctx_r_2], #16] \n\t" + /* Store r^4 */ +- "STP w11, w12, [%[ctx_r_4], #0] \n\t" +- "STP w13, w14, [%[ctx_r_4], #8] \n\t" +- "STR w15, [%[ctx_r_4], #16] \n\t" +- ++ "ORR x19, x19, x20, LSL #32 \n\t" ++ "ORR x21, x21, x22, LSL #32 \n\t" ++ "STP x19, x21, [%[ctx_r_4]] \n\t" ++ "STR w23, [%[ctx_r_4], #16] \n\t" + /* h (accumulator) = 0 */ + "STP xzr, xzr, [%[ctx_h_0]] \n\t" + "STR wzr, [%[ctx_h_0], #16] \n\t" ++ /* Save pad for later */ ++ "STP x10, x11, [%[ctx_pad]] \n\t" + /* Zero leftover */ + "STR xzr, [%[ctx_leftover]] \n\t" + /* Zero finished */ +@@ -1034,7 +1063,6 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz) + : + : [clamp] "r" (clamp), + [key] "r" (key), +- [ctx_r64] "r" (ctx->r64), + [ctx_r] "r" (ctx->r), + [ctx_r_2] "r" (ctx->r_2), + [ctx_r_4] "r" (ctx->r_4), +@@ -1043,8 +1071,9 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz) + [ctx_leftover] "r" (&ctx->leftover), + [ctx_finished] "r" (&ctx->finished) + : "memory", "cc", +- "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", +- "x19", "x20" ++ "w7", "w14", "w15", "w16", "w17", "w19", "w20", "w21", "w22", "w23", ++ "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", ++ "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" + ); + + return 0; +@@ -1053,6 +1082,7 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz) + + int wc_Poly1305Final(Poly1305* ctx, byte* mac) + { ++ + if (ctx == NULL) + return BAD_FUNC_ARG; + +@@ -1067,55 +1097,68 @@ int wc_Poly1305Final(Poly1305* ctx, byte* mac) + } + + __asm__ __volatile__ ( +- "LDP x9, x10, %[ctx_pad] \n\t" +- /* Load h */ +- "LDP w4, w5, [%[ctx_h], #0] \n\t" +- "LDP w6, w7, [%[ctx_h], #8] \n\t" +- "LDR w8, [%[ctx_h], #16] \n\t" ++ /* Load raw h and zero h registers */ ++ "LDP x2, x3, %[h_addr] \n\t" ++ "MOV x5, xzr \n\t" ++ "LDR w4, %[h_4_addr] \n\t" ++ "MOV x6, xzr \n\t" ++ "LDP x16, x17, %[pad_addr] \n\t" + /* Base 26 -> Base 64 */ +- "ORR x4, x4, x5, LSL #26\n\t" +- "ORR x4, x4, x6, LSL #52\n\t" +- "LSR x5, x6, #12\n\t" +- "ORR x5, x5, x7, LSL #14\n\t" +- "ORR x5, x5, x8, LSL #40\n\t" +- "LSR x6, x8, #24\n\t" ++ "MOV w5, w2 \n\t" ++ "LSR x2, x2, #32 \n\t" ++ "ORR x5, x5, x2, LSL #26 \n\t" ++ "ORR x5, x5, x3, LSL #52 \n\t" ++ "LSR w6, w3, #12 \n\t" ++ "LSR x3, x3, #32 \n\t" ++ "ORR x6, x6, x3, LSL #14 \n\t" ++ "ORR x6, x6, x4, LSL #40 \n\t" ++ "LSR x7, x4, #24 \n\t" + /* Check if h is larger than p */ +- "ADDS x1, x4, #5 \n\t" +- "ADCS x2, x5, xzr \n\t" +- "ADC x3, x6, xzr \n\t" ++ "ADDS x2, x5, #5 \n\t" ++ "ADCS x3, x6, xzr \n\t" ++ "ADC x4, x7, xzr \n\t" + /* Check if h+5 is larger than 2^130 */ +- "CMP x3, #3 \n\t" +- "CSEL x4, x1, x4, HI \n\t" ++ "CMP x4, #3 \n\t" + "CSEL x5, x2, x5, HI \n\t" +- "ADDS x4, x4, x9 \n\t" +- "ADC x5, x5, x10 \n\t" +- "STP x4, x5, [%[mac]] \n\t" +- +- /* Zero out h */ +- "STP xzr, xzr, [%[ctx_h]] \n\t" +- "STR wzr, [%[ctx_h], #16] \n\t" +- /* Zero out r64 */ +- "STP xzr, xzr, [%[ctx_r64]] \n\t" +- /* Zero out r */ +- "STP xzr, xzr, [%[ctx_r]] \n\t" +- "STR wzr, [%[ctx_r], #16] \n\t" +- /* Zero out r_2 */ +- "STP xzr, xzr, [%[ctx_r_2]] \n\t" +- "STR wzr, [%[ctx_r_2], #16] \n\t" +- /* Zero out r_4 */ +- "STP xzr, xzr, [%[ctx_r_4]] \n\t" +- "STR wzr, [%[ctx_r_4], #16] \n\t" +- /* Zero out pad */ +- "STP xzr, xzr, %[ctx_pad] \n\t" +- : +- : [ctx_pad] "m" (ctx->pad), [ctx_h] "r" (ctx->h), [mac] "r" (mac), +- [ctx_r64] "r" (ctx->r64), [ctx_r] "r" (ctx->r), +- [ctx_r_2] "r" (ctx->r_2), [ctx_r_4] "r" (ctx->r_4) ++ "CSEL x6, x3, x6, HI \n\t" ++ "ADDS x5, x5, x16 \n\t" ++ "ADC x6, x6, x17 \n\t" ++ "STP x5, x6, [%[mac]] \n\t" ++ : [mac] "+r" (mac) ++ : [pad_addr] "m" (ctx->pad), ++ [h_addr] "m" (ctx->h), ++ [h_4_addr] "m" (ctx->h[4]) + : "memory", "cc", +- "w4", "w5", "w6", "w7", "w8", +- "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10" ++ "w2", "w3", "w4", "w5", "w6", "w7", "x2", "x3", "x4", "x5", ++ "x6", "x7", "x16", "x17" + ); + ++ /* zero out the state */ ++ ctx->h[0] = 0; ++ ctx->h[1] = 0; ++ ctx->h[2] = 0; ++ ctx->h[3] = 0; ++ ctx->h[4] = 0; ++ ctx->r[0] = 0; ++ ctx->r[1] = 0; ++ ctx->r[2] = 0; ++ ctx->r[3] = 0; ++ ctx->r[4] = 0; ++ ctx->r_2[0] = 0; ++ ctx->r_2[1] = 0; ++ ctx->r_2[2] = 0; ++ ctx->r_2[3] = 0; ++ ctx->r_2[4] = 0; ++ ctx->r_4[0] = 0; ++ ctx->r_4[1] = 0; ++ ctx->r_4[2] = 0; ++ ctx->r_4[3] = 0; ++ ctx->r_4[4] = 0; ++ ctx->pad[0] = 0; ++ ctx->pad[1] = 0; ++ ctx->pad[2] = 0; ++ ctx->pad[3] = 0; ++ + return 0; + } + +diff --git a/wolfssl/wolfcrypt/poly1305.h b/wolfssl/wolfcrypt/poly1305.h +index d4db48762..6b3751fef 100644 +--- a/wolfssl/wolfcrypt/poly1305.h ++++ b/wolfssl/wolfcrypt/poly1305.h +@@ -89,7 +89,6 @@ typedef struct Poly1305 { + unsigned char finished; + unsigned char started; + #elif defined(WOLFSSL_ARMASM) && defined(__aarch64__) +- ALIGN128 word64 r64[2]; + ALIGN128 word32 r[5]; + ALIGN128 word32 r_2[5]; /* r^2 */ + ALIGN128 word32 r_4[5]; /* r^4 */ +-- +2.47.1 +