diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 4f88a806c0..6c259f0d37 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -1079,6 +1079,7 @@ WOLFSSL_SNIFFER_NO_RECOVERY WOLFSSL_SP_ARM32_UDIV WOLFSSL_SP_FAST_NCT_EXPTMOD WOLFSSL_SP_INT_SQR_VOLATILE +WOLFSSL_SP_LADDER_MODEXP WOLFSSL_SSLKEYLOGFILE_USE_ENV WOLFSSL_STACK_CHECK WOLFSSL_STM32C5 diff --git a/wolfcrypt/src/sp_c32.c b/wolfcrypt/src/sp_c32.c index 3dbf45846f..098a505397 100644 --- a/wolfcrypt/src/sp_c32.c +++ b/wolfcrypt/src/sp_c32.c @@ -254,8 +254,13 @@ sp_uint64 __muldi3(sp_uint64 a, sp_uint64 b) #define NEED_ADDR_MASK #endif #if defined(WOLFSSL_HAVE_SP_RSA) || defined(WOLFSSL_HAVE_SP_DH) - #if !defined(WC_NO_CACHE_RESISTANT) && \ - (defined(WOLFSSL_HAVE_SP_DH) || !defined(WOLFSSL_RSA_PUBLIC_ONLY)) + /* Only the ladder implementations of mod_exp use the address mask: the + * small one, and the one WOLFSSL_SP_LADDER_MODEXP selects in place of the + * default cache-resistant window. The windows index a table instead. */ + #if (defined(WOLFSSL_HAVE_SP_DH) || !defined(WOLFSSL_RSA_PUBLIC_ONLY)) && \ + ((defined(WOLFSSL_SP_SMALL) && !defined(WOLFSSL_SP_FAST_MODEXP)) || \ + (defined(WOLFSSL_SP_LADDER_MODEXP) && !defined(WC_NO_HARDEN) && \ + !defined(WC_NO_CACHE_RESISTANT))) #define NEED_ADDR_MASK #endif #endif @@ -2321,6 +2326,40 @@ static int sp_2048_mod_36(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_2048_div_36(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_36(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 72; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 72; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -2413,8 +2452,95 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 72); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 72, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 72); + } + rt = td + (17 * 72); + sq = td + (18 * 72); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_36(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_36(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_36(t[0], t[0], norm); + err = sp_2048_mod_36(t[0], t[0], m); + } + } + else { + sp_2048_mul_36(t[0], a, norm); + err = sp_2048_mod_36(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_36(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_2048_mont_mul_36(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 72); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_2048_mont_sqr_36(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_36(rt, rt, m, mp); + } + sp_2048_mont_mul_36(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_36(rt, m, mp); + n = sp_2048_cmp_36(rt, m); + sp_2048_cond_sub_36(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 72); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 72); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -2489,6 +2615,125 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 72) + 144); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 72) + 144, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 72; + rt = td + 1152; + ct = td + 1224; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_36(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_36(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_36(t[1], t[1], norm); + err = sp_2048_mod_36(t[1], t[1], m); + } + } + else { + sp_2048_mul_36(t[1], a, norm); + err = sp_2048_mod_36(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_36(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_36(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_36(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_36(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_36(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_36(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_36(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_36(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_36(t[10], t[ 5], m, mp); + sp_2048_mont_mul_36(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_36(t[12], t[ 6], m, mp); + sp_2048_mont_mul_36(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_36(t[14], t[ 7], m, mp); + sp_2048_mont_mul_36(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 36) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_2048_get_from_table_36(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; + } + + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + + sp_2048_get_from_table_36(ct, t, y); + sp_2048_mont_mul_36(rt, rt, ct, m, mp); + } + + sp_2048_mont_reduce_36(rt, m, mp); + n = sp_2048_cmp_36(rt, m); + sp_2048_cond_sub_36(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 72); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 72) + 72); sp_digit* t[32]; @@ -3359,6 +3604,40 @@ static int sp_2048_mod_72(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_72(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 144; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 144; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -3451,8 +3730,95 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 144); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 144, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 144); + } + rt = td + (9 * 144); + sq = td + (10 * 144); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_72(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_72(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_72(t[0], t[0], norm); + err = sp_2048_mod_72(t[0], t[0], m); + } + } + else { + sp_2048_mul_72(t[0], a, norm); + err = sp_2048_mod_72(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_72(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_2048_mont_mul_72(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 144); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_2048_mont_sqr_72(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_72(rt, rt, m, mp); + } + sp_2048_mont_mul_72(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_72(rt, m, mp); + n = sp_2048_cmp_72(rt, m); + sp_2048_cond_sub_72(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 144); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 144); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -3528,9 +3894,10 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (16 * 144) + 144); + SP_DECL_VAR(sp_digit, td, (16 * 144) + 288); sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -3543,13 +3910,14 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (16 * 144) + 144, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 144) + 288, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; for (i=0; i<16; i++) t[i] = td + i * 144; rt = td + 2304; + ct = td + 2448; sp_2048_mont_setup(m, &mp); sp_2048_mont_norm_72(norm, m); @@ -3603,7 +3971,7 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, y = (int)((n >> 28) & 0xf); n = (sp_uint32)n << 4; c -= 4; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 144); + sp_2048_get_from_table_72(rt, t, y); while ((i >= 0) || (c >= 4)) { if (c >= 4) { y = (byte)((n >> 28) & 0xf); @@ -3630,7 +3998,8 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, sp_2048_mont_sqr_72(rt, rt, m, mp); sp_2048_mont_sqr_72(rt, rt, m, mp); - sp_2048_mont_mul_72(rt, rt, t[y], m, mp); + sp_2048_get_from_table_72(ct, t, y); + sp_2048_mont_mul_72(rt, rt, ct, m, mp); } sp_2048_mont_reduce_72(rt, m, mp); @@ -3643,19 +4012,135 @@ static int sp_2048_mod_exp_72(sp_digit* r, const sp_digit* a, return err; #endif -} -#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ - !defined(WOLFSSL_SP_FAST_MODEXP) -/* Non-blocking modular exponentiation. State machine driven by sp_ctx; - * each call advances one Montgomery op or one bit-extract step then - * returns MP_WOULDBLOCK until the final reduction completes. - */ -typedef struct sp_2048_mod_exp_72_ctx { - int state; - sp_digit td[3 * 144]; - sp_digit* t[3]; - sp_digit* norm; - sp_digit mp; +#else + SP_DECL_VAR(sp_digit, td, (16 * 144) + 144); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 144) + 144, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 144; + rt = td + 2304; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_72(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_72(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_72(t[1], t[1], norm); + err = sp_2048_mod_72(t[1], t[1], m); + } + } + else { + sp_2048_mul_72(t[1], a, norm); + err = sp_2048_mod_72(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_72(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_72(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_72(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_72(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_72(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_72(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_72(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_72(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_72(t[10], t[ 5], m, mp); + sp_2048_mont_mul_72(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_72(t[12], t[ 6], m, mp); + sp_2048_mont_mul_72(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_72(t[14], t[ 7], m, mp); + sp_2048_mont_mul_72(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 72) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 144); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; + } + + sp_2048_mont_sqr_72(rt, rt, m, mp); + sp_2048_mont_sqr_72(rt, rt, m, mp); + sp_2048_mont_sqr_72(rt, rt, m, mp); + sp_2048_mont_sqr_72(rt, rt, m, mp); + + sp_2048_mont_mul_72(rt, rt, t[y], m, mp); + } + + sp_2048_mont_reduce_72(rt, m, mp); + n = sp_2048_cmp_72(rt, m); + sp_2048_cond_sub_72(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 144); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} +#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ + !defined(WOLFSSL_SP_FAST_MODEXP) +/* Non-blocking modular exponentiation. State machine driven by sp_ctx; + * each call advances one Montgomery op or one bit-extract step then + * returns MP_WOULDBLOCK until the final reduction completes. + */ +typedef struct sp_2048_mod_exp_72_ctx { + int state; + sp_digit td[3 * 144]; + sp_digit* t[3]; + sp_digit* norm; + sp_digit mp; sp_digit n; int i; int c; @@ -6323,6 +6808,40 @@ static int sp_3072_mod_53(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_3072_div_53(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_53(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 106; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 106; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -6415,8 +6934,95 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 106); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 106, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 106); + } + rt = td + (17 * 106); + sq = td + (18 * 106); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_53(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_53(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_53(t[0], t[0], norm); + err = sp_3072_mod_53(t[0], t[0], m); + } + } + else { + sp_3072_mul_53(t[0], a, norm); + err = sp_3072_mod_53(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_53(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_3072_mont_mul_53(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 106); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_3072_mont_sqr_53(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_53(rt, rt, m, mp); + } + sp_3072_mont_mul_53(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_53(rt, m, mp); + n = sp_3072_cmp_53(rt, m); + sp_3072_cond_sub_53(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 106); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 106); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -6492,9 +7098,10 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (32 * 106) + 106); - sp_digit* t[32]; + SP_DECL_VAR(sp_digit, td, (16 * 106) + 212); + sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -6507,13 +7114,14 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (32 * 106) + 106, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 106) + 212, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<32; i++) + for (i=0; i<16; i++) t[i] = td + i * 106; - rt = td + 3392; + rt = td + 1696; + ct = td + 1802; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_53(norm, m); @@ -6546,24 +7154,8 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, sp_3072_mont_mul_53(t[13], t[ 7], t[ 6], m, mp); sp_3072_mont_sqr_53(t[14], t[ 7], m, mp); sp_3072_mont_mul_53(t[15], t[ 8], t[ 7], m, mp); - sp_3072_mont_sqr_53(t[16], t[ 8], m, mp); - sp_3072_mont_mul_53(t[17], t[ 9], t[ 8], m, mp); - sp_3072_mont_sqr_53(t[18], t[ 9], m, mp); - sp_3072_mont_mul_53(t[19], t[10], t[ 9], m, mp); - sp_3072_mont_sqr_53(t[20], t[10], m, mp); - sp_3072_mont_mul_53(t[21], t[11], t[10], m, mp); - sp_3072_mont_sqr_53(t[22], t[11], m, mp); - sp_3072_mont_mul_53(t[23], t[12], t[11], m, mp); - sp_3072_mont_sqr_53(t[24], t[12], m, mp); - sp_3072_mont_mul_53(t[25], t[13], t[12], m, mp); - sp_3072_mont_sqr_53(t[26], t[13], m, mp); - sp_3072_mont_mul_53(t[27], t[14], t[13], m, mp); - sp_3072_mont_sqr_53(t[28], t[14], m, mp); - sp_3072_mont_mul_53(t[29], t[15], t[14], m, mp); - sp_3072_mont_sqr_53(t[30], t[15], m, mp); - sp_3072_mont_mul_53(t[31], t[16], t[15], m, mp); - bits = ((bits + 4) / 5) * 5; + bits = ((bits + 3) / 4) * 4; i = ((bits + 28) / 29) - 1; c = bits % 29; if (c == 0) { @@ -6576,30 +7168,30 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, n = 0; i--; } - if (c < 5) { + if (c < 4) { n |= (sp_uint32)e[i--] << (3 - c); c += 29; } - y = (int)((n >> 27) & 0x1f); - n = (sp_uint32)n << 5; - c -= 5; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 106); - while ((i >= 0) || (c >= 5)) { - if (c >= 5) { - y = (byte)((n >> 27) & 0x1f); - n = (sp_uint32)n << 5; - c -= 5; + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_3072_get_from_table_53(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; } else if (c == 0) { n = (sp_uint32)e[i--] << 3; - y = (byte)((n >> 27) & 0x1f); - n = (sp_uint32)n << 5; - c = 24; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; } else { - y = (byte)((n >> 27) & 0x1f); + y = (byte)((n >> 28) & 0xf); n = (sp_uint32)e[i--] << 3; - c = 5 - c; + c = 4 - c; y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); n = (sp_uint32)n << c; c = 29 - c; @@ -6609,9 +7201,9 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, sp_3072_mont_sqr_53(rt, rt, m, mp); sp_3072_mont_sqr_53(rt, rt, m, mp); sp_3072_mont_sqr_53(rt, rt, m, mp); - sp_3072_mont_sqr_53(rt, rt, m, mp); - sp_3072_mont_mul_53(rt, rt, t[y], m, mp); + sp_3072_get_from_table_53(ct, t, y); + sp_3072_mont_mul_53(rt, rt, ct, m, mp); } sp_3072_mont_reduce_53(rt, m, mp); @@ -6624,12 +7216,145 @@ static int sp_3072_mod_exp_53(sp_digit* r, const sp_digit* a, return err; #endif -} - -#endif /* (WOLFSSL_HAVE_SP_RSA & !WOLFSSL_RSA_PUBLIC_ONLY) | WOLFSSL_HAVE_SP_DH */ - -/* Sub b from a into r. (r = a - b) - * +#else + SP_DECL_VAR(sp_digit, td, (32 * 106) + 106); + sp_digit* t[32]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (32 * 106) + 106, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<32; i++) + t[i] = td + i * 106; + rt = td + 3392; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_53(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_53(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_53(t[1], t[1], norm); + err = sp_3072_mod_53(t[1], t[1], m); + } + } + else { + sp_3072_mul_53(t[1], a, norm); + err = sp_3072_mod_53(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_53(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_53(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_53(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_53(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_53(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_53(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_53(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_53(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_53(t[10], t[ 5], m, mp); + sp_3072_mont_mul_53(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_53(t[12], t[ 6], m, mp); + sp_3072_mont_mul_53(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_53(t[14], t[ 7], m, mp); + sp_3072_mont_mul_53(t[15], t[ 8], t[ 7], m, mp); + sp_3072_mont_sqr_53(t[16], t[ 8], m, mp); + sp_3072_mont_mul_53(t[17], t[ 9], t[ 8], m, mp); + sp_3072_mont_sqr_53(t[18], t[ 9], m, mp); + sp_3072_mont_mul_53(t[19], t[10], t[ 9], m, mp); + sp_3072_mont_sqr_53(t[20], t[10], m, mp); + sp_3072_mont_mul_53(t[21], t[11], t[10], m, mp); + sp_3072_mont_sqr_53(t[22], t[11], m, mp); + sp_3072_mont_mul_53(t[23], t[12], t[11], m, mp); + sp_3072_mont_sqr_53(t[24], t[12], m, mp); + sp_3072_mont_mul_53(t[25], t[13], t[12], m, mp); + sp_3072_mont_sqr_53(t[26], t[13], m, mp); + sp_3072_mont_mul_53(t[27], t[14], t[13], m, mp); + sp_3072_mont_sqr_53(t[28], t[14], m, mp); + sp_3072_mont_mul_53(t[29], t[15], t[14], m, mp); + sp_3072_mont_sqr_53(t[30], t[15], m, mp); + sp_3072_mont_mul_53(t[31], t[16], t[15], m, mp); + + bits = ((bits + 4) / 5) * 5; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 53) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 5) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 27) & 0x1f); + n = (sp_uint32)n << 5; + c -= 5; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 106); + while ((i >= 0) || (c >= 5)) { + if (c >= 5) { + y = (byte)((n >> 27) & 0x1f); + n = (sp_uint32)n << 5; + c -= 5; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 27) & 0x1f); + n = (sp_uint32)n << 5; + c = 24; + } + else { + y = (byte)((n >> 27) & 0x1f); + n = (sp_uint32)e[i--] << 3; + c = 5 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; + } + + sp_3072_mont_sqr_53(rt, rt, m, mp); + sp_3072_mont_sqr_53(rt, rt, m, mp); + sp_3072_mont_sqr_53(rt, rt, m, mp); + sp_3072_mont_sqr_53(rt, rt, m, mp); + sp_3072_mont_sqr_53(rt, rt, m, mp); + + sp_3072_mont_mul_53(rt, rt, t[y], m, mp); + } + + sp_3072_mont_reduce_53(rt, m, mp); + n = sp_3072_cmp_53(rt, m); + sp_3072_cond_sub_53(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 106); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} + +#endif /* (WOLFSSL_HAVE_SP_RSA & !WOLFSSL_RSA_PUBLIC_ONLY) | WOLFSSL_HAVE_SP_DH */ + +/* Sub b from a into r. (r = a - b) + * * @param [out] r A single precision integer. * @param [in] a A single precision integer. * @param [in] b A single precision integer. @@ -7135,6 +7860,40 @@ static int sp_3072_mod_106(sp_digit* r, const sp_digit* a, const sp_digit* m) } #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_106(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 212; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 212; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -7227,8 +7986,95 @@ static int sp_3072_mod_exp_106(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 212); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 212, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 212); + } + rt = td + (9 * 212); + sq = td + (10 * 212); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_106(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_106(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_106(t[0], t[0], norm); + err = sp_3072_mod_106(t[0], t[0], m); + } + } + else { + sp_3072_mul_106(t[0], a, norm); + err = sp_3072_mod_106(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_106(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_3072_mont_mul_106(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 212); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_3072_mont_sqr_106(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_106(rt, rt, m, mp); + } + sp_3072_mont_mul_106(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_106(rt, m, mp); + n = sp_3072_cmp_106(rt, m); + sp_3072_cond_sub_106(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 212); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 212); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -7303,6 +8149,125 @@ static int sp_3072_mod_exp_106(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 212) + 424); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 212) + 424, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 212; + rt = td + 3392; + ct = td + 3604; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_106(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_106(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_106(t[1], t[1], norm); + err = sp_3072_mod_106(t[1], t[1], m); + } + } + else { + sp_3072_mul_106(t[1], a, norm); + err = sp_3072_mod_106(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_106(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_106(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_106(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_106(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_106(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_106(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_106(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_106(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_106(t[10], t[ 5], m, mp); + sp_3072_mont_mul_106(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_106(t[12], t[ 6], m, mp); + sp_3072_mont_mul_106(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_106(t[14], t[ 7], m, mp); + sp_3072_mont_mul_106(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 106) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_3072_get_from_table_106(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; + } + + sp_3072_mont_sqr_106(rt, rt, m, mp); + sp_3072_mont_sqr_106(rt, rt, m, mp); + sp_3072_mont_sqr_106(rt, rt, m, mp); + sp_3072_mont_sqr_106(rt, rt, m, mp); + + sp_3072_get_from_table_106(ct, t, y); + sp_3072_mont_mul_106(rt, rt, ct, m, mp); + } + + sp_3072_mont_reduce_106(rt, m, mp); + n = sp_3072_cmp_106(rt, m); + sp_3072_cond_sub_106(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 212); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 212) + 212); sp_digit* t[16]; @@ -10673,6 +11638,40 @@ static int sp_3072_mod_56(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_3072_div_56(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_56(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 112; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 112; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -10710,7 +11709,169 @@ static int sp_3072_mod_exp_56(sp_digit* r, const sp_digit* a, norm = td; for (i=0; i<3; i++) { t[i] = td + (i * 56 * 2); - XMEMSET(t[i], 0, sizeof(sp_digit) * 56U * 2U); + XMEMSET(t[i], 0, sizeof(sp_digit) * 56U * 2U); + } + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_56(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_56(t[1], a, m); + } + else { + XMEMCPY(t[1], a, sizeof(sp_digit) * 56U); + } + } + if (err == MP_OKAY) { + sp_3072_mul_56(t[1], t[1], norm); + err = sp_3072_mod_56(t[1], t[1], m); + } + + if (err == MP_OKAY) { + i = (bits - 1) / 28; + c = ((bits - 1) % 28) + 1; + n = (sp_uint32)e[i--] << (28 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 28; + } + + y = (int)((n >> 27) & 1); + n = (sp_uint32)n << 1; + + sp_3072_mont_mul_56(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 56 * 2); + sp_3072_mont_sqr_56(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 56 * 2); + } + + sp_3072_mont_reduce_56(t[0], m, mp); + n = sp_3072_cmp_56(t[0], m); + sp_3072_cond_sub_56(t[0], t[0], m, (sp_digit)~(n >> 31)); + XMEMCPY(r, t[0], sizeof(*r) * 56 * 2); + + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 112); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 112, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 112); + } + rt = td + (17 * 112); + sq = td + (18 * 112); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_56(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_56(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_56(t[0], t[0], norm); + err = sp_3072_mod_56(t[0], t[0], m); + } + } + else { + sp_3072_mul_56(t[0], a, norm); + err = sp_3072_mod_56(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_56(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_3072_mont_mul_56(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 112); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 28] >> (i % 28)) & 1) == 0) { + sp_3072_mont_sqr_56(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 28] >> (l % 28)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 28] >> (j % 28)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_56(rt, rt, m, mp); + } + sp_3072_mont_mul_56(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_56(rt, m, mp); + n = sp_3072_cmp_56(rt, m); + sp_3072_cond_sub_56(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 112); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP + SP_DECL_VAR(sp_digit, td, 3 * 112); + sp_digit* t[3] = {0, 0, 0}; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 3 * 112, NULL, DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<3; i++) { + t[i] = td + (i * 56 * 2); } sp_3072_mont_setup(m, &mp); @@ -10718,15 +11879,16 @@ static int sp_3072_mod_exp_56(sp_digit* r, const sp_digit* a, if (reduceA != 0) { err = sp_3072_mod_56(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_56(t[1], t[1], norm); + err = sp_3072_mod_56(t[1], t[1], m); + } } else { - XMEMCPY(t[1], a, sizeof(sp_digit) * 56U); + sp_3072_mul_56(t[1], a, norm); + err = sp_3072_mod_56(t[1], t[1], m); } } - if (err == MP_OKAY) { - sp_3072_mul_56(t[1], t[1], norm); - err = sp_3072_mod_56(t[1], t[1], m); - } if (err == MP_OKAY) { i = (bits - 1) / 28; @@ -10760,15 +11922,16 @@ static int sp_3072_mod_exp_56(sp_digit* r, const sp_digit* a, n = sp_3072_cmp_56(t[0], m); sp_3072_cond_sub_56(t[0], t[0], m, (sp_digit)~(n >> 31)); XMEMCPY(r, t[0], sizeof(*r) * 56 * 2); - } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; -#elif !defined(WC_NO_CACHE_RESISTANT) - SP_DECL_VAR(sp_digit, td, 3 * 112); - sp_digit* t[3] = {0, 0, 0}; +#else + SP_DECL_VAR(sp_digit, td, (16 * 112) + 224); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -10781,12 +11944,14 @@ static int sp_3072_mod_exp_56(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, 3 * 112, NULL, DYNAMIC_TYPE_TMP_BUFFER); + SP_ALLOC_VAR(sp_digit, td, (16 * 112) + 224, NULL, + DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<3; i++) { - t[i] = td + (i * 56 * 2); - } + for (i=0; i<16; i++) + t[i] = td + i * 112; + rt = td + 1792; + ct = td + 1904; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_56(norm, m); @@ -10805,42 +11970,82 @@ static int sp_3072_mod_exp_56(sp_digit* r, const sp_digit* a, } if (err == MP_OKAY) { - i = (bits - 1) / 28; - c = ((bits - 1) % 28) + 1; - n = (sp_uint32)e[i--] << (28 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } + sp_3072_mont_sqr_56(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_56(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_56(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_56(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_56(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_56(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_56(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_56(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_56(t[10], t[ 5], m, mp); + sp_3072_mont_mul_56(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_56(t[12], t[ 6], m, mp); + sp_3072_mont_mul_56(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_56(t[14], t[ 7], m, mp); + sp_3072_mont_mul_56(t[15], t[ 8], t[ 7], m, mp); - n = e[i--]; - c = 28; + bits = ((bits + 3) / 4) * 4; + i = ((bits + 27) / 28) - 1; + c = bits % 28; + if (c == 0) { + c = 28; + } + if (i < 56) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (4 - c); + c += 28; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_3072_get_from_table_56(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 4; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 24; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 4; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 28 - c; } - y = (int)((n >> 27) & 1); - n = (sp_uint32)n << 1; - - sp_3072_mont_mul_56(t[y^1], t[0], t[1], m, mp); + sp_3072_mont_sqr_56(rt, rt, m, mp); + sp_3072_mont_sqr_56(rt, rt, m, mp); + sp_3072_mont_sqr_56(rt, rt, m, mp); + sp_3072_mont_sqr_56(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 56 * 2); - sp_3072_mont_sqr_56(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 56 * 2); + sp_3072_get_from_table_56(ct, t, y); + sp_3072_mont_mul_56(rt, rt, ct, m, mp); } - sp_3072_mont_reduce_56(t[0], m, mp); - n = sp_3072_cmp_56(t[0], m); - sp_3072_cond_sub_56(t[0], t[0], m, (sp_digit)~(n >> 31)); - XMEMCPY(r, t[0], sizeof(*r) * 56 * 2); + sp_3072_mont_reduce_56(rt, m, mp); + n = sp_3072_cmp_56(rt, m); + sp_3072_cond_sub_56(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 112); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 112) + 112); sp_digit* t[32]; @@ -11567,6 +12772,40 @@ static int sp_3072_mod_112(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_112(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 224; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 224; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -11604,7 +12843,169 @@ static int sp_3072_mod_exp_112(sp_digit* r, const sp_digit* a, norm = td; for (i=0; i<3; i++) { t[i] = td + (i * 112 * 2); - XMEMSET(t[i], 0, sizeof(sp_digit) * 112U * 2U); + XMEMSET(t[i], 0, sizeof(sp_digit) * 112U * 2U); + } + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_112(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_112(t[1], a, m); + } + else { + XMEMCPY(t[1], a, sizeof(sp_digit) * 112U); + } + } + if (err == MP_OKAY) { + sp_3072_mul_112(t[1], t[1], norm); + err = sp_3072_mod_112(t[1], t[1], m); + } + + if (err == MP_OKAY) { + i = (bits - 1) / 28; + c = ((bits - 1) % 28) + 1; + n = (sp_uint32)e[i--] << (28 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 28; + } + + y = (int)((n >> 27) & 1); + n = (sp_uint32)n << 1; + + sp_3072_mont_mul_112(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 112 * 2); + sp_3072_mont_sqr_112(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 112 * 2); + } + + sp_3072_mont_reduce_112(t[0], m, mp); + n = sp_3072_cmp_112(t[0], m); + sp_3072_cond_sub_112(t[0], t[0], m, (sp_digit)~(n >> 31)); + XMEMCPY(r, t[0], sizeof(*r) * 112 * 2); + + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 224); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 224, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 224); + } + rt = td + (9 * 224); + sq = td + (10 * 224); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_112(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_112(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_112(t[0], t[0], norm); + err = sp_3072_mod_112(t[0], t[0], m); + } + } + else { + sp_3072_mul_112(t[0], a, norm); + err = sp_3072_mod_112(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_112(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_3072_mont_mul_112(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 224); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 28] >> (i % 28)) & 1) == 0) { + sp_3072_mont_sqr_112(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 28] >> (l % 28)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 28] >> (j % 28)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_112(rt, rt, m, mp); + } + sp_3072_mont_mul_112(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_112(rt, m, mp); + n = sp_3072_cmp_112(rt, m); + sp_3072_cond_sub_112(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 224); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP + SP_DECL_VAR(sp_digit, td, 3 * 224); + sp_digit* t[3] = {0, 0, 0}; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 3 * 224, NULL, DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<3; i++) { + t[i] = td + (i * 112 * 2); } sp_3072_mont_setup(m, &mp); @@ -11612,15 +13013,16 @@ static int sp_3072_mod_exp_112(sp_digit* r, const sp_digit* a, if (reduceA != 0) { err = sp_3072_mod_112(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_112(t[1], t[1], norm); + err = sp_3072_mod_112(t[1], t[1], m); + } } else { - XMEMCPY(t[1], a, sizeof(sp_digit) * 112U); + sp_3072_mul_112(t[1], a, norm); + err = sp_3072_mod_112(t[1], t[1], m); } } - if (err == MP_OKAY) { - sp_3072_mul_112(t[1], t[1], norm); - err = sp_3072_mod_112(t[1], t[1], m); - } if (err == MP_OKAY) { i = (bits - 1) / 28; @@ -11654,15 +13056,16 @@ static int sp_3072_mod_exp_112(sp_digit* r, const sp_digit* a, n = sp_3072_cmp_112(t[0], m); sp_3072_cond_sub_112(t[0], t[0], m, (sp_digit)~(n >> 31)); XMEMCPY(r, t[0], sizeof(*r) * 112 * 2); - } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; -#elif !defined(WC_NO_CACHE_RESISTANT) - SP_DECL_VAR(sp_digit, td, 3 * 224); - sp_digit* t[3] = {0, 0, 0}; +#else + SP_DECL_VAR(sp_digit, td, (16 * 224) + 448); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -11675,12 +13078,14 @@ static int sp_3072_mod_exp_112(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, 3 * 224, NULL, DYNAMIC_TYPE_TMP_BUFFER); + SP_ALLOC_VAR(sp_digit, td, (16 * 224) + 448, NULL, + DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<3; i++) { - t[i] = td + (i * 112 * 2); - } + for (i=0; i<16; i++) + t[i] = td + i * 224; + rt = td + 3584; + ct = td + 3808; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_112(norm, m); @@ -11699,42 +13104,82 @@ static int sp_3072_mod_exp_112(sp_digit* r, const sp_digit* a, } if (err == MP_OKAY) { - i = (bits - 1) / 28; - c = ((bits - 1) % 28) + 1; - n = (sp_uint32)e[i--] << (28 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } + sp_3072_mont_sqr_112(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_112(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_112(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_112(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_112(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_112(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_112(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_112(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_112(t[10], t[ 5], m, mp); + sp_3072_mont_mul_112(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_112(t[12], t[ 6], m, mp); + sp_3072_mont_mul_112(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_112(t[14], t[ 7], m, mp); + sp_3072_mont_mul_112(t[15], t[ 8], t[ 7], m, mp); - n = e[i--]; - c = 28; + bits = ((bits + 3) / 4) * 4; + i = ((bits + 27) / 28) - 1; + c = bits % 28; + if (c == 0) { + c = 28; + } + if (i < 112) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (4 - c); + c += 28; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_3072_get_from_table_112(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 4; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 24; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 4; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 28 - c; } - y = (int)((n >> 27) & 1); - n = (sp_uint32)n << 1; - - sp_3072_mont_mul_112(t[y^1], t[0], t[1], m, mp); + sp_3072_mont_sqr_112(rt, rt, m, mp); + sp_3072_mont_sqr_112(rt, rt, m, mp); + sp_3072_mont_sqr_112(rt, rt, m, mp); + sp_3072_mont_sqr_112(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 112 * 2); - sp_3072_mont_sqr_112(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 112 * 2); + sp_3072_get_from_table_112(ct, t, y); + sp_3072_mont_mul_112(rt, rt, ct, m, mp); } - sp_3072_mont_reduce_112(t[0], m, mp); - n = sp_3072_cmp_112(t[0], m); - sp_3072_cond_sub_112(t[0], t[0], m, (sp_digit)~(n >> 31)); - XMEMCPY(r, t[0], sizeof(*r) * 112 * 2); + sp_3072_mont_reduce_112(rt, m, mp); + n = sp_3072_cmp_112(rt, m); + sp_3072_cond_sub_112(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 224); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 224) + 224); sp_digit* t[16]; @@ -14203,6 +15648,40 @@ static int sp_4096_mod_71(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_4096_div_71(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_71(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 142; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 142; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -14295,8 +15774,95 @@ static int sp_4096_mod_exp_71(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 142); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 142, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 142); + } + rt = td + (17 * 142); + sq = td + (18 * 142); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_71(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_71(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_71(t[0], t[0], norm); + err = sp_4096_mod_71(t[0], t[0], m); + } + } + else { + sp_4096_mul_71(t[0], a, norm); + err = sp_4096_mod_71(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_71(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_4096_mont_mul_71(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 142); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_4096_mont_sqr_71(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_71(rt, rt, m, mp); + } + sp_4096_mont_mul_71(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_71(rt, m, mp); + n = sp_4096_cmp_71(rt, m); + sp_4096_cond_sub_71(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 142); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 142); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -14311,12 +15877,91 @@ static int sp_4096_mod_exp_71(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, 3 * 142, NULL, DYNAMIC_TYPE_TMP_BUFFER); + SP_ALLOC_VAR(sp_digit, td, 3 * 142, NULL, DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<3; i++) { + t[i] = td + (i * 71 * 2); + } + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_71(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_71(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_71(t[1], t[1], norm); + err = sp_4096_mod_71(t[1], t[1], m); + } + } + else { + sp_4096_mul_71(t[1], a, norm); + err = sp_4096_mod_71(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + i = (bits - 1) / 29; + c = ((bits - 1) % 29) + 1; + n = (sp_uint32)e[i--] << (29 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 29; + } + + y = (int)((n >> 28) & 1); + n = (sp_uint32)n << 1; + + sp_4096_mont_mul_71(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 71 * 2); + sp_4096_mont_sqr_71(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 71 * 2); + } + + sp_4096_mont_reduce_71(t[0], m, mp); + n = sp_4096_cmp_71(t[0], m); + sp_4096_cond_sub_71(t[0], t[0], m, (sp_digit)~(n >> 31)); + XMEMCPY(r, t[0], sizeof(*r) * 71 * 2); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 142) + 284); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 142) + 284, NULL, + DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<3; i++) { - t[i] = td + (i * 71 * 2); - } + for (i=0; i<16; i++) + t[i] = td + i * 142; + rt = td + 2272; + ct = td + 2414; sp_4096_mont_setup(m, &mp); sp_4096_mont_norm_71(norm, m); @@ -14335,42 +15980,82 @@ static int sp_4096_mod_exp_71(sp_digit* r, const sp_digit* a, } if (err == MP_OKAY) { - i = (bits - 1) / 29; - c = ((bits - 1) % 29) + 1; - n = (sp_uint32)e[i--] << (29 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } + sp_4096_mont_sqr_71(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_71(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_71(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_71(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_71(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_71(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_71(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_71(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_71(t[10], t[ 5], m, mp); + sp_4096_mont_mul_71(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_71(t[12], t[ 6], m, mp); + sp_4096_mont_mul_71(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_71(t[14], t[ 7], m, mp); + sp_4096_mont_mul_71(t[15], t[ 8], t[ 7], m, mp); - n = e[i--]; - c = 29; + bits = ((bits + 3) / 4) * 4; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 71) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_4096_get_from_table_71(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; } - y = (int)((n >> 28) & 1); - n = (sp_uint32)n << 1; - - sp_4096_mont_mul_71(t[y^1], t[0], t[1], m, mp); + sp_4096_mont_sqr_71(rt, rt, m, mp); + sp_4096_mont_sqr_71(rt, rt, m, mp); + sp_4096_mont_sqr_71(rt, rt, m, mp); + sp_4096_mont_sqr_71(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 71 * 2); - sp_4096_mont_sqr_71(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 71 * 2); + sp_4096_get_from_table_71(ct, t, y); + sp_4096_mont_mul_71(rt, rt, ct, m, mp); } - sp_4096_mont_reduce_71(t[0], m, mp); - n = sp_4096_cmp_71(t[0], m); - sp_4096_cond_sub_71(t[0], t[0], m, (sp_digit)~(n >> 31)); - XMEMCPY(r, t[0], sizeof(*r) * 71 * 2); + sp_4096_mont_reduce_71(rt, m, mp); + n = sp_4096_cmp_71(rt, m); + sp_4096_cond_sub_71(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 142); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 142) + 142); sp_digit* t[32]; @@ -15016,6 +16701,40 @@ static int sp_4096_mod_142(sp_digit* r, const sp_digit* a, const sp_digit* m) } #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_142(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 284; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 284; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -15108,8 +16827,95 @@ static int sp_4096_mod_exp_142(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 284); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 284, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 284); + } + rt = td + (9 * 284); + sq = td + (10 * 284); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_142(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_142(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_142(t[0], t[0], norm); + err = sp_4096_mod_142(t[0], t[0], m); + } + } + else { + sp_4096_mul_142(t[0], a, norm); + err = sp_4096_mod_142(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_142(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_4096_mont_mul_142(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 284); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 29] >> (i % 29)) & 1) == 0) { + sp_4096_mont_sqr_142(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 29] >> (l % 29)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 29] >> (j % 29)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_142(rt, rt, m, mp); + } + sp_4096_mont_mul_142(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_142(rt, m, mp); + n = sp_4096_cmp_142(rt, m); + sp_4096_cond_sub_142(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 284); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 284); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -15142,48 +16948,167 @@ static int sp_4096_mod_exp_142(sp_digit* r, const sp_digit* a, } } else { - sp_4096_mul_142(t[1], a, norm); - err = sp_4096_mod_142(t[1], t[1], m); + sp_4096_mul_142(t[1], a, norm); + err = sp_4096_mod_142(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + i = (bits - 1) / 29; + c = ((bits - 1) % 29) + 1; + n = (sp_uint32)e[i--] << (29 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 29; + } + + y = (int)((n >> 28) & 1); + n = (sp_uint32)n << 1; + + sp_4096_mont_mul_142(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 142 * 2); + sp_4096_mont_sqr_142(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 142 * 2); + } + + sp_4096_mont_reduce_142(t[0], m, mp); + n = sp_4096_cmp_142(t[0], m); + sp_4096_cond_sub_142(t[0], t[0], m, (sp_digit)~(n >> 31)); + XMEMCPY(r, t[0], sizeof(*r) * 142 * 2); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 284) + 568); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 284) + 568, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 284; + rt = td + 4544; + ct = td + 4828; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_142(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_142(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_142(t[1], t[1], norm); + err = sp_4096_mod_142(t[1], t[1], m); + } + } + else { + sp_4096_mul_142(t[1], a, norm); + err = sp_4096_mod_142(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_142(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_142(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_142(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_142(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_142(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_142(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_142(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_142(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_142(t[10], t[ 5], m, mp); + sp_4096_mont_mul_142(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_142(t[12], t[ 6], m, mp); + sp_4096_mont_mul_142(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_142(t[14], t[ 7], m, mp); + sp_4096_mont_mul_142(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 28) / 29) - 1; + c = bits % 29; + if (c == 0) { + c = 29; + } + if (i < 142) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; } - } - - if (err == MP_OKAY) { - i = (bits - 1) / 29; - c = ((bits - 1) % 29) + 1; - n = (sp_uint32)e[i--] << (29 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } - - n = e[i--]; - c = 29; + if (c < 4) { + n |= (sp_uint32)e[i--] << (3 - c); + c += 29; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_4096_get_from_table_142(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 3; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 25; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 29 - c; } - y = (int)((n >> 28) & 1); - n = (sp_uint32)n << 1; - - sp_4096_mont_mul_142(t[y^1], t[0], t[1], m, mp); + sp_4096_mont_sqr_142(rt, rt, m, mp); + sp_4096_mont_sqr_142(rt, rt, m, mp); + sp_4096_mont_sqr_142(rt, rt, m, mp); + sp_4096_mont_sqr_142(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 142 * 2); - sp_4096_mont_sqr_142(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 142 * 2); + sp_4096_get_from_table_142(ct, t, y); + sp_4096_mont_mul_142(rt, rt, ct, m, mp); } - sp_4096_mont_reduce_142(t[0], m, mp); - n = sp_4096_cmp_142(t[0], m); - sp_4096_cond_sub_142(t[0], t[0], m, (sp_digit)~(n >> 31)); - XMEMCPY(r, t[0], sizeof(*r) * 142 * 2); + sp_4096_mont_reduce_142(rt, m, mp); + n = sp_4096_cmp_142(rt, m); + sp_4096_cond_sub_142(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 284); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 284) + 284); sp_digit* t[16]; @@ -18473,6 +20398,40 @@ static int sp_4096_mod_81(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_4096_div_81(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_81(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 162; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 162; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -18565,8 +20524,95 @@ static int sp_4096_mod_exp_81(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 162); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 162, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 162); + } + rt = td + (17 * 162); + sq = td + (18 * 162); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_81(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_81(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_81(t[0], t[0], norm); + err = sp_4096_mod_81(t[0], t[0], m); + } + } + else { + sp_4096_mul_81(t[0], a, norm); + err = sp_4096_mod_81(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_81(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_4096_mont_mul_81(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 162); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 26] >> (i % 26)) & 1) == 0) { + sp_4096_mont_sqr_81(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 26] >> (l % 26)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 26] >> (j % 26)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_81(rt, rt, m, mp); + } + sp_4096_mont_mul_81(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_81(rt, m, mp); + n = sp_4096_cmp_81(rt, m); + sp_4096_cond_sub_81(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 162); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 162); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -18587,60 +20633,179 @@ static int sp_4096_mod_exp_81(sp_digit* r, const sp_digit* a, for (i=0; i<3; i++) { t[i] = td + (i * 81 * 2); } - - sp_4096_mont_setup(m, &mp); - sp_4096_mont_norm_81(norm, m); - - if (reduceA != 0) { - err = sp_4096_mod_81(t[1], a, m); - if (err == MP_OKAY) { - sp_4096_mul_81(t[1], t[1], norm); - err = sp_4096_mod_81(t[1], t[1], m); - } + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_81(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_81(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_81(t[1], t[1], norm); + err = sp_4096_mod_81(t[1], t[1], m); + } + } + else { + sp_4096_mul_81(t[1], a, norm); + err = sp_4096_mod_81(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + i = (bits - 1) / 26; + c = ((bits - 1) % 26) + 1; + n = (sp_uint32)e[i--] << (26 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 26; + } + + y = (int)((n >> 25) & 1); + n = (sp_uint32)n << 1; + + sp_4096_mont_mul_81(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 81 * 2); + sp_4096_mont_sqr_81(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 81 * 2); + } + + sp_4096_mont_reduce_81(t[0], m, mp); + n = sp_4096_cmp_81(t[0], m); + sp_4096_cond_sub_81(t[0], t[0], m, (sp_digit)~(n >> 31)); + XMEMCPY(r, t[0], sizeof(*r) * 81 * 2); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 162) + 324); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 162) + 324, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 162; + rt = td + 2592; + ct = td + 2754; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_81(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_81(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_81(t[1], t[1], norm); + err = sp_4096_mod_81(t[1], t[1], m); + } + } + else { + sp_4096_mul_81(t[1], a, norm); + err = sp_4096_mod_81(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_81(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_81(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_81(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_81(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_81(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_81(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_81(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_81(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_81(t[10], t[ 5], m, mp); + sp_4096_mont_mul_81(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_81(t[12], t[ 6], m, mp); + sp_4096_mont_mul_81(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_81(t[14], t[ 7], m, mp); + sp_4096_mont_mul_81(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 25) / 26) - 1; + c = bits % 26; + if (c == 0) { + c = 26; + } + if (i < 81) { + n = (sp_uint32)e[i--] << (32 - c); } else { - sp_4096_mul_81(t[1], a, norm); - err = sp_4096_mod_81(t[1], t[1], m); + n = 0; + i--; } - } - - if (err == MP_OKAY) { - i = (bits - 1) / 26; - c = ((bits - 1) % 26) + 1; - n = (sp_uint32)e[i--] << (26 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } - - n = e[i--]; - c = 26; + if (c < 4) { + n |= (sp_uint32)e[i--] << (6 - c); + c += 26; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_4096_get_from_table_81(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 6; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 22; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 6; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 26 - c; } - y = (int)((n >> 25) & 1); - n = (sp_uint32)n << 1; - - sp_4096_mont_mul_81(t[y^1], t[0], t[1], m, mp); + sp_4096_mont_sqr_81(rt, rt, m, mp); + sp_4096_mont_sqr_81(rt, rt, m, mp); + sp_4096_mont_sqr_81(rt, rt, m, mp); + sp_4096_mont_sqr_81(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 81 * 2); - sp_4096_mont_sqr_81(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 81 * 2); + sp_4096_get_from_table_81(ct, t, y); + sp_4096_mont_mul_81(rt, rt, ct, m, mp); } - sp_4096_mont_reduce_81(t[0], m, mp); - n = sp_4096_cmp_81(t[0], m); - sp_4096_cond_sub_81(t[0], t[0], m, (sp_digit)~(n >> 31)); - XMEMCPY(r, t[0], sizeof(*r) * 81 * 2); + sp_4096_mont_reduce_81(rt, m, mp); + n = sp_4096_cmp_81(rt, m); + sp_4096_cond_sub_81(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 162); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 162) + 162); sp_digit* t[32]; @@ -19354,6 +21519,40 @@ static int sp_4096_mod_162(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_162(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint32 mask; + sp_uint32 diff; + + for (j = 0; j < 324; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint32)(e ^ idx); + diff = (sp_uint32)((diff | (sp_uint32)((sp_uint32)0 - diff)) >> + (32 - 1)); + mask = (sp_uint32)((sp_uint32)0 - ((sp_uint32)1 - diff)); + + for (j = 0; j < 324; j++) { + r[j] |= (sp_digit)((sp_uint32)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -19446,8 +21645,95 @@ static int sp_4096_mod_exp_162(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 324); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 324, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 324); + } + rt = td + (9 * 324); + sq = td + (10 * 324); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_162(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_162(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_162(t[0], t[0], norm); + err = sp_4096_mod_162(t[0], t[0], m); + } + } + else { + sp_4096_mul_162(t[0], a, norm); + err = sp_4096_mod_162(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_162(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_4096_mont_mul_162(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 324); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 26] >> (i % 26)) & 1) == 0) { + sp_4096_mont_sqr_162(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 26] >> (l % 26)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 26] >> (j % 26)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_162(rt, rt, m, mp); + } + sp_4096_mont_mul_162(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_162(rt, m, mp); + n = sp_4096_cmp_162(rt, m); + sp_4096_cond_sub_162(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(*r) * 324); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 324); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -19522,6 +21808,125 @@ static int sp_4096_mod_exp_162(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 324) + 648); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 324) + 648, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 324; + rt = td + 5184; + ct = td + 5508; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_162(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_162(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_162(t[1], t[1], norm); + err = sp_4096_mod_162(t[1], t[1], m); + } + } + else { + sp_4096_mul_162(t[1], a, norm); + err = sp_4096_mod_162(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_162(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_162(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_162(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_162(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_162(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_162(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_162(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_162(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_162(t[10], t[ 5], m, mp); + sp_4096_mont_mul_162(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_162(t[12], t[ 6], m, mp); + sp_4096_mont_mul_162(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_162(t[14], t[ 7], m, mp); + sp_4096_mont_mul_162(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 25) / 26) - 1; + c = bits % 26; + if (c == 0) { + c = 26; + } + if (i < 162) { + n = (sp_uint32)e[i--] << (32 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint32)e[i--] << (6 - c); + c += 26; + } + y = (int)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + sp_4096_get_from_table_162(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint32)e[i--] << 6; + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)n << 4; + c = 22; + } + else { + y = (byte)((n >> 28) & 0xf); + n = (sp_uint32)e[i--] << 6; + c = 4 - c; + y |= (byte)((n >> (32 - c)) & ((1 << c) - 1)); + n = (sp_uint32)n << c; + c = 26 - c; + } + + sp_4096_mont_sqr_162(rt, rt, m, mp); + sp_4096_mont_sqr_162(rt, rt, m, mp); + sp_4096_mont_sqr_162(rt, rt, m, mp); + sp_4096_mont_sqr_162(rt, rt, m, mp); + + sp_4096_get_from_table_162(ct, t, y); + sp_4096_mont_mul_162(rt, rt, ct, m, mp); + } + + sp_4096_mont_reduce_162(rt, m, mp); + n = sp_4096_cmp_162(rt, m); + sp_4096_cond_sub_162(rt, rt, m, (sp_digit)~(n >> 31)); + XMEMCPY(r, rt, sizeof(sp_digit) * 324); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 324) + 324); sp_digit* t[16]; diff --git a/wolfcrypt/src/sp_c64.c b/wolfcrypt/src/sp_c64.c index 3ef4549c47..92fd867e15 100644 --- a/wolfcrypt/src/sp_c64.c +++ b/wolfcrypt/src/sp_c64.c @@ -180,8 +180,13 @@ #define NEED_ADDR_MASK #endif #if defined(WOLFSSL_HAVE_SP_RSA) || defined(WOLFSSL_HAVE_SP_DH) - #if !defined(WC_NO_CACHE_RESISTANT) && \ - (defined(WOLFSSL_HAVE_SP_DH) || !defined(WOLFSSL_RSA_PUBLIC_ONLY)) + /* Only the ladder implementations of mod_exp use the address mask: the + * small one, and the one WOLFSSL_SP_LADDER_MODEXP selects in place of the + * default cache-resistant window. The windows index a table instead. */ + #if (defined(WOLFSSL_HAVE_SP_DH) || !defined(WOLFSSL_RSA_PUBLIC_ONLY)) && \ + ((defined(WOLFSSL_SP_SMALL) && !defined(WOLFSSL_SP_FAST_MODEXP)) || \ + (defined(WOLFSSL_SP_LADDER_MODEXP) && !defined(WC_NO_HARDEN) && \ + !defined(WC_NO_CACHE_RESISTANT))) #define NEED_ADDR_MASK #endif #endif @@ -1158,6 +1163,40 @@ static int sp_2048_mod_17(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_2048_div_17(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_17(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 34; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 34; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -1250,8 +1289,95 @@ static int sp_2048_mod_exp_17(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 34); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 34, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 34); + } + rt = td + (17 * 34); + sq = td + (18 * 34); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_17(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_17(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_17(t[0], t[0], norm); + err = sp_2048_mod_17(t[0], t[0], m); + } + } + else { + sp_2048_mul_17(t[0], a, norm); + err = sp_2048_mod_17(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_17(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_2048_mont_mul_17(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 34); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 61] >> (i % 61)) & 1) == 0) { + sp_2048_mont_sqr_17(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 61] >> (l % 61)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 61] >> (j % 61)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_17(rt, rt, m, mp); + } + sp_2048_mont_mul_17(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_17(rt, m, mp); + n = sp_2048_cmp_17(rt, m); + sp_2048_cond_sub_17(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 34); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 34); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -1326,6 +1452,125 @@ static int sp_2048_mod_exp_17(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 34) + 68); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 34) + 68, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 34; + rt = td + 544; + ct = td + 578; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_17(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_17(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_17(t[1], t[1], norm); + err = sp_2048_mod_17(t[1], t[1], m); + } + } + else { + sp_2048_mul_17(t[1], a, norm); + err = sp_2048_mod_17(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_17(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_17(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_17(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_17(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_17(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_17(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_17(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_17(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_17(t[10], t[ 5], m, mp); + sp_2048_mont_mul_17(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_17(t[12], t[ 6], m, mp); + sp_2048_mont_mul_17(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_17(t[14], t[ 7], m, mp); + sp_2048_mont_mul_17(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 60) / 61) - 1; + c = bits % 61; + if (c == 0) { + c = 61; + } + if (i < 17) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (3 - c); + c += 61; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_2048_get_from_table_17(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 3; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 57; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 61 - c; + } + + sp_2048_mont_sqr_17(rt, rt, m, mp); + sp_2048_mont_sqr_17(rt, rt, m, mp); + sp_2048_mont_sqr_17(rt, rt, m, mp); + sp_2048_mont_sqr_17(rt, rt, m, mp); + + sp_2048_get_from_table_17(ct, t, y); + sp_2048_mont_mul_17(rt, rt, ct, m, mp); + } + + sp_2048_mont_reduce_17(rt, m, mp); + n = sp_2048_cmp_17(rt, m); + sp_2048_cond_sub_17(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 34); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 34) + 34); sp_digit* t[32]; @@ -1956,6 +2201,40 @@ static int sp_2048_mod_34(sp_digit* r, const sp_digit* a, const sp_digit* m) } #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_34(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 68; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 68; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -2048,8 +2327,95 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 68); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 68, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 68); + } + rt = td + (9 * 68); + sq = td + (10 * 68); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_34(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_34(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_34(t[0], t[0], norm); + err = sp_2048_mod_34(t[0], t[0], m); + } + } + else { + sp_2048_mul_34(t[0], a, norm); + err = sp_2048_mod_34(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_34(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_2048_mont_mul_34(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 68); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 61] >> (i % 61)) & 1) == 0) { + sp_2048_mont_sqr_34(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 61] >> (l % 61)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 61] >> (j % 61)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_34(rt, rt, m, mp); + } + sp_2048_mont_mul_34(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_34(rt, m, mp); + n = sp_2048_cmp_34(rt, m); + sp_2048_cond_sub_34(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 68); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 68); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -2125,9 +2491,10 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (16 * 68) + 68); + SP_DECL_VAR(sp_digit, td, (16 * 68) + 136); sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -2140,13 +2507,14 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (16 * 68) + 68, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 68) + 136, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; for (i=0; i<16; i++) t[i] = td + i * 68; rt = td + 1088; + ct = td + 1156; sp_2048_mont_setup(m, &mp); sp_2048_mont_norm_34(norm, m); @@ -2200,7 +2568,7 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, y = (int)((n >> 60) & 0xf); n = (sp_uint64)n << 4; c -= 4; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 68); + sp_2048_get_from_table_34(rt, t, y); while ((i >= 0) || (c >= 4)) { if (c >= 4) { y = (byte)((n >> 60) & 0xf); @@ -2227,7 +2595,8 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, sp_2048_mont_sqr_34(rt, rt, m, mp); sp_2048_mont_sqr_34(rt, rt, m, mp); - sp_2048_mont_mul_34(rt, rt, t[y], m, mp); + sp_2048_get_from_table_34(ct, t, y); + sp_2048_mont_mul_34(rt, rt, ct, m, mp); } sp_2048_mont_reduce_34(rt, m, mp); @@ -2240,18 +2609,134 @@ static int sp_2048_mod_exp_34(sp_digit* r, const sp_digit* a, return err; #endif -} -#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ - !defined(WOLFSSL_SP_FAST_MODEXP) -/* Non-blocking modular exponentiation. State machine driven by sp_ctx; - * each call advances one Montgomery op or one bit-extract step then - * returns MP_WOULDBLOCK until the final reduction completes. - */ -typedef struct sp_2048_mod_exp_34_ctx { - int state; - sp_digit td[3 * 68]; - sp_digit* t[3]; - sp_digit* norm; +#else + SP_DECL_VAR(sp_digit, td, (16 * 68) + 68); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 68) + 68, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 68; + rt = td + 1088; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_34(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_34(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_34(t[1], t[1], norm); + err = sp_2048_mod_34(t[1], t[1], m); + } + } + else { + sp_2048_mul_34(t[1], a, norm); + err = sp_2048_mod_34(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_34(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_34(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_34(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_34(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_34(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_34(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_34(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_34(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_34(t[10], t[ 5], m, mp); + sp_2048_mont_mul_34(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_34(t[12], t[ 6], m, mp); + sp_2048_mont_mul_34(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_34(t[14], t[ 7], m, mp); + sp_2048_mont_mul_34(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 60) / 61) - 1; + c = bits % 61; + if (c == 0) { + c = 61; + } + if (i < 34) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (3 - c); + c += 61; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 68); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 3; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 57; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 3; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 61 - c; + } + + sp_2048_mont_sqr_34(rt, rt, m, mp); + sp_2048_mont_sqr_34(rt, rt, m, mp); + sp_2048_mont_sqr_34(rt, rt, m, mp); + sp_2048_mont_sqr_34(rt, rt, m, mp); + + sp_2048_mont_mul_34(rt, rt, t[y], m, mp); + } + + sp_2048_mont_reduce_34(rt, m, mp); + n = sp_2048_cmp_34(rt, m); + sp_2048_cond_sub_34(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 68); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} +#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ + !defined(WOLFSSL_SP_FAST_MODEXP) +/* Non-blocking modular exponentiation. State machine driven by sp_ctx; + * each call advances one Montgomery op or one bit-extract step then + * returns MP_WOULDBLOCK until the final reduction completes. + */ +typedef struct sp_2048_mod_exp_34_ctx { + int state; + sp_digit td[3 * 68]; + sp_digit* t[3]; + sp_digit* norm; sp_digit mp; sp_digit n; int i; @@ -5009,6 +5494,40 @@ static int sp_2048_mod_18(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_2048_div_18(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_18(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 36; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 36; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -5101,8 +5620,95 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 36); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 36, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 36); + } + rt = td + (17 * 36); + sq = td + (18 * 36); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_18(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_18(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_18(t[0], t[0], norm); + err = sp_2048_mod_18(t[0], t[0], m); + } + } + else { + sp_2048_mul_18(t[0], a, norm); + err = sp_2048_mod_18(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_18(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_2048_mont_mul_18(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 36); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 57] >> (i % 57)) & 1) == 0) { + sp_2048_mont_sqr_18(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 57] >> (l % 57)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 57] >> (j % 57)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_18(rt, rt, m, mp); + } + sp_2048_mont_mul_18(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_18(rt, m, mp); + n = sp_2048_cmp_18(rt, m); + sp_2048_cond_sub_18(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 36); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 36); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -5178,9 +5784,10 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (32 * 36) + 36); - sp_digit* t[32]; + SP_DECL_VAR(sp_digit, td, (16 * 36) + 72); + sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -5193,13 +5800,14 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (32 * 36) + 36, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 36) + 72, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<32; i++) + for (i=0; i<16; i++) t[i] = td + i * 36; - rt = td + 1152; + rt = td + 576; + ct = td + 612; sp_2048_mont_setup(m, &mp); sp_2048_mont_norm_18(norm, m); @@ -5232,24 +5840,8 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, sp_2048_mont_mul_18(t[13], t[ 7], t[ 6], m, mp); sp_2048_mont_sqr_18(t[14], t[ 7], m, mp); sp_2048_mont_mul_18(t[15], t[ 8], t[ 7], m, mp); - sp_2048_mont_sqr_18(t[16], t[ 8], m, mp); - sp_2048_mont_mul_18(t[17], t[ 9], t[ 8], m, mp); - sp_2048_mont_sqr_18(t[18], t[ 9], m, mp); - sp_2048_mont_mul_18(t[19], t[10], t[ 9], m, mp); - sp_2048_mont_sqr_18(t[20], t[10], m, mp); - sp_2048_mont_mul_18(t[21], t[11], t[10], m, mp); - sp_2048_mont_sqr_18(t[22], t[11], m, mp); - sp_2048_mont_mul_18(t[23], t[12], t[11], m, mp); - sp_2048_mont_sqr_18(t[24], t[12], m, mp); - sp_2048_mont_mul_18(t[25], t[13], t[12], m, mp); - sp_2048_mont_sqr_18(t[26], t[13], m, mp); - sp_2048_mont_mul_18(t[27], t[14], t[13], m, mp); - sp_2048_mont_sqr_18(t[28], t[14], m, mp); - sp_2048_mont_mul_18(t[29], t[15], t[14], m, mp); - sp_2048_mont_sqr_18(t[30], t[15], m, mp); - sp_2048_mont_mul_18(t[31], t[16], t[15], m, mp); - bits = ((bits + 4) / 5) * 5; + bits = ((bits + 3) / 4) * 4; i = ((bits + 56) / 57) - 1; c = bits % 57; if (c == 0) { @@ -5262,30 +5854,30 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, n = 0; i--; } - if (c < 5) { + if (c < 4) { n |= (sp_uint64)e[i--] << (7 - c); c += 57; } - y = (int)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 36); - while ((i >= 0) || (c >= 5)) { - if (c >= 5) { - y = (byte)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_2048_get_from_table_18(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; } else if (c == 0) { n = (sp_uint64)e[i--] << 7; - y = (byte)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c = 52; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 53; } else { - y = (byte)((n >> 59) & 0x1f); + y = (byte)((n >> 60) & 0xf); n = (sp_uint64)e[i--] << 7; - c = 5 - c; + c = 4 - c; y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); n = (sp_uint64)n << c; c = 57 - c; @@ -5295,9 +5887,9 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, sp_2048_mont_sqr_18(rt, rt, m, mp); sp_2048_mont_sqr_18(rt, rt, m, mp); sp_2048_mont_sqr_18(rt, rt, m, mp); - sp_2048_mont_sqr_18(rt, rt, m, mp); - sp_2048_mont_mul_18(rt, rt, t[y], m, mp); + sp_2048_get_from_table_18(ct, t, y); + sp_2048_mont_mul_18(rt, rt, ct, m, mp); } sp_2048_mont_reduce_18(rt, m, mp); @@ -5310,12 +5902,145 @@ static int sp_2048_mod_exp_18(sp_digit* r, const sp_digit* a, return err; #endif -} - -#endif /* (WOLFSSL_HAVE_SP_RSA & !WOLFSSL_RSA_PUBLIC_ONLY) | WOLFSSL_HAVE_SP_DH */ - -/* r = 2^n mod m where n is the number of bits to reduce by. - * Given m must be 2048 bits, just need to subtract. +#else + SP_DECL_VAR(sp_digit, td, (32 * 36) + 36); + sp_digit* t[32]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (32 * 36) + 36, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<32; i++) + t[i] = td + i * 36; + rt = td + 1152; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_18(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_18(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_18(t[1], t[1], norm); + err = sp_2048_mod_18(t[1], t[1], m); + } + } + else { + sp_2048_mul_18(t[1], a, norm); + err = sp_2048_mod_18(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_18(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_18(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_18(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_18(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_18(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_18(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_18(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_18(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_18(t[10], t[ 5], m, mp); + sp_2048_mont_mul_18(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_18(t[12], t[ 6], m, mp); + sp_2048_mont_mul_18(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_18(t[14], t[ 7], m, mp); + sp_2048_mont_mul_18(t[15], t[ 8], t[ 7], m, mp); + sp_2048_mont_sqr_18(t[16], t[ 8], m, mp); + sp_2048_mont_mul_18(t[17], t[ 9], t[ 8], m, mp); + sp_2048_mont_sqr_18(t[18], t[ 9], m, mp); + sp_2048_mont_mul_18(t[19], t[10], t[ 9], m, mp); + sp_2048_mont_sqr_18(t[20], t[10], m, mp); + sp_2048_mont_mul_18(t[21], t[11], t[10], m, mp); + sp_2048_mont_sqr_18(t[22], t[11], m, mp); + sp_2048_mont_mul_18(t[23], t[12], t[11], m, mp); + sp_2048_mont_sqr_18(t[24], t[12], m, mp); + sp_2048_mont_mul_18(t[25], t[13], t[12], m, mp); + sp_2048_mont_sqr_18(t[26], t[13], m, mp); + sp_2048_mont_mul_18(t[27], t[14], t[13], m, mp); + sp_2048_mont_sqr_18(t[28], t[14], m, mp); + sp_2048_mont_mul_18(t[29], t[15], t[14], m, mp); + sp_2048_mont_sqr_18(t[30], t[15], m, mp); + sp_2048_mont_mul_18(t[31], t[16], t[15], m, mp); + + bits = ((bits + 4) / 5) * 5; + i = ((bits + 56) / 57) - 1; + c = bits % 57; + if (c == 0) { + c = 57; + } + if (i < 18) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 5) { + n |= (sp_uint64)e[i--] << (7 - c); + c += 57; + } + y = (int)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 36); + while ((i >= 0) || (c >= 5)) { + if (c >= 5) { + y = (byte)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 7; + y = (byte)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c = 52; + } + else { + y = (byte)((n >> 59) & 0x1f); + n = (sp_uint64)e[i--] << 7; + c = 5 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 57 - c; + } + + sp_2048_mont_sqr_18(rt, rt, m, mp); + sp_2048_mont_sqr_18(rt, rt, m, mp); + sp_2048_mont_sqr_18(rt, rt, m, mp); + sp_2048_mont_sqr_18(rt, rt, m, mp); + sp_2048_mont_sqr_18(rt, rt, m, mp); + + sp_2048_mont_mul_18(rt, rt, t[y], m, mp); + } + + sp_2048_mont_reduce_18(rt, m, mp); + n = sp_2048_cmp_18(rt, m); + sp_2048_cond_sub_18(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 36); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} + +#endif /* (WOLFSSL_HAVE_SP_RSA & !WOLFSSL_RSA_PUBLIC_ONLY) | WOLFSSL_HAVE_SP_DH */ + +/* r = 2^n mod m where n is the number of bits to reduce by. + * Given m must be 2048 bits, just need to subtract. * * @param [out] r A single precision number. * @param [in] m A single precision number. @@ -5870,6 +6595,40 @@ static int sp_2048_mod_36(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_2048_get_from_table_36(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 72; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 72; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -5962,8 +6721,95 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 72); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 72, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 72); + } + rt = td + (9 * 72); + sq = td + (10 * 72); + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_36(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_36(t[0], a, m); + if (err == MP_OKAY) { + sp_2048_mul_36(t[0], t[0], norm); + err = sp_2048_mod_36(t[0], t[0], m); + } + } + else { + sp_2048_mul_36(t[0], a, norm); + err = sp_2048_mod_36(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_2048_mont_sqr_36(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_2048_mont_mul_36(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 72); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 57] >> (i % 57)) & 1) == 0) { + sp_2048_mont_sqr_36(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 57] >> (l % 57)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 57] >> (j % 57)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_2048_mont_sqr_36(rt, rt, m, mp); + } + sp_2048_mont_mul_36(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_2048_mont_reduce_36(rt, m, mp); + n = sp_2048_cmp_36(rt, m); + sp_2048_cond_sub_36(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 72); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 72); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -6039,9 +6885,10 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (16 * 72) + 72); + SP_DECL_VAR(sp_digit, td, (16 * 72) + 144); sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -6054,13 +6901,14 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (16 * 72) + 72, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 72) + 144, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; for (i=0; i<16; i++) t[i] = td + i * 72; rt = td + 1152; + ct = td + 1224; sp_2048_mont_setup(m, &mp); sp_2048_mont_norm_36(norm, m); @@ -6114,7 +6962,7 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, y = (int)((n >> 60) & 0xf); n = (sp_uint64)n << 4; c -= 4; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 72); + sp_2048_get_from_table_36(rt, t, y); while ((i >= 0) || (c >= 4)) { if (c >= 4) { y = (byte)((n >> 60) & 0xf); @@ -6141,7 +6989,8 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, sp_2048_mont_sqr_36(rt, rt, m, mp); sp_2048_mont_sqr_36(rt, rt, m, mp); - sp_2048_mont_mul_36(rt, rt, t[y], m, mp); + sp_2048_get_from_table_36(ct, t, y); + sp_2048_mont_mul_36(rt, rt, ct, m, mp); } sp_2048_mont_reduce_36(rt, m, mp); @@ -6154,51 +7003,167 @@ static int sp_2048_mod_exp_36(sp_digit* r, const sp_digit* a, return err; #endif -} -#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ - !defined(WOLFSSL_SP_FAST_MODEXP) -/* Non-blocking modular exponentiation. State machine driven by sp_ctx; - * each call advances one Montgomery op or one bit-extract step then - * returns MP_WOULDBLOCK until the final reduction completes. - */ -typedef struct sp_2048_mod_exp_36_ctx { - int state; - sp_digit td[3 * 72]; - sp_digit* t[3]; - sp_digit* norm; - sp_digit mp; +#else + SP_DECL_VAR(sp_digit, td, (16 * 72) + 72); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; sp_digit n; int i; int c; byte y; - int reduceA; - int bits; -} sp_2048_mod_exp_36_ctx; + int err = MP_OKAY; -/* Modular exponentiate a to the e mod m. (r = a^e mod m) - * - * Non-blocking version. Call repeatedly with the same context until it does - * not return MP_WOULDBLOCK. State is saved and restored through ctx. - * - * @param [in, out] ctx Context saving state for the non-blocking - * operation. - * @param [out] r A single precision number that is the result of the - * operation. - * @param [in] a A single precision number being exponentiated. - * @param [in] e A single precision number that is the exponent. - * @param [in] bits The number of bits in the exponent. - * @param [in] m A single precision number that is the modulus. - * @param [in] reduceA Whether to reduce a modulo m before the operation. - * - * @return MP_OKAY on completion. - * @return MP_WOULDBLOCK while more work remains. - */ -static int sp_2048_mod_exp_36_nb(sp_2048_mod_exp_36_ctx* ctx, - sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, - const sp_digit* m, int reduceA) -{ - int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); - int j; + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 72) + 72, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 72; + rt = td + 1152; + + sp_2048_mont_setup(m, &mp); + sp_2048_mont_norm_36(norm, m); + + if (reduceA != 0) { + err = sp_2048_mod_36(t[1], a, m); + if (err == MP_OKAY) { + sp_2048_mul_36(t[1], t[1], norm); + err = sp_2048_mod_36(t[1], t[1], m); + } + } + else { + sp_2048_mul_36(t[1], a, norm); + err = sp_2048_mod_36(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_2048_mont_sqr_36(t[ 2], t[ 1], m, mp); + sp_2048_mont_mul_36(t[ 3], t[ 2], t[ 1], m, mp); + sp_2048_mont_sqr_36(t[ 4], t[ 2], m, mp); + sp_2048_mont_mul_36(t[ 5], t[ 3], t[ 2], m, mp); + sp_2048_mont_sqr_36(t[ 6], t[ 3], m, mp); + sp_2048_mont_mul_36(t[ 7], t[ 4], t[ 3], m, mp); + sp_2048_mont_sqr_36(t[ 8], t[ 4], m, mp); + sp_2048_mont_mul_36(t[ 9], t[ 5], t[ 4], m, mp); + sp_2048_mont_sqr_36(t[10], t[ 5], m, mp); + sp_2048_mont_mul_36(t[11], t[ 6], t[ 5], m, mp); + sp_2048_mont_sqr_36(t[12], t[ 6], m, mp); + sp_2048_mont_mul_36(t[13], t[ 7], t[ 6], m, mp); + sp_2048_mont_sqr_36(t[14], t[ 7], m, mp); + sp_2048_mont_mul_36(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 56) / 57) - 1; + c = bits % 57; + if (c == 0) { + c = 57; + } + if (i < 36) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (7 - c); + c += 57; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 72); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 7; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 53; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 7; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 57 - c; + } + + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + sp_2048_mont_sqr_36(rt, rt, m, mp); + + sp_2048_mont_mul_36(rt, rt, t[y], m, mp); + } + + sp_2048_mont_reduce_36(rt, m, mp); + n = sp_2048_cmp_36(rt, m); + sp_2048_cond_sub_36(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 72); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} +#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ + !defined(WOLFSSL_SP_FAST_MODEXP) +/* Non-blocking modular exponentiation. State machine driven by sp_ctx; + * each call advances one Montgomery op or one bit-extract step then + * returns MP_WOULDBLOCK until the final reduction completes. + */ +typedef struct sp_2048_mod_exp_36_ctx { + int state; + sp_digit td[3 * 72]; + sp_digit* t[3]; + sp_digit* norm; + sp_digit mp; + sp_digit n; + int i; + int c; + byte y; + int reduceA; + int bits; +} sp_2048_mod_exp_36_ctx; + +/* Modular exponentiate a to the e mod m. (r = a^e mod m) + * + * Non-blocking version. Call repeatedly with the same context until it does + * not return MP_WOULDBLOCK. State is saved and restored through ctx. + * + * @param [in, out] ctx Context saving state for the non-blocking + * operation. + * @param [out] r A single precision number that is the result of the + * operation. + * @param [in] a A single precision number being exponentiated. + * @param [in] e A single precision number that is the exponent. + * @param [in] bits The number of bits in the exponent. + * @param [in] m A single precision number that is the modulus. + * @param [in] reduceA Whether to reduce a modulo m before the operation. + * + * @return MP_OKAY on completion. + * @return MP_WOULDBLOCK while more work remains. + */ +static int sp_2048_mod_exp_36_nb(sp_2048_mod_exp_36_ctx* ctx, + sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, + const sp_digit* m, int reduceA) +{ + int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); + int j; switch (ctx->state) { case 0: /* INIT: layout td, mont_setup, mont_norm */ @@ -8258,6 +9223,40 @@ static int sp_3072_mod_26(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_3072_div_26(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_26(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 52; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 52; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -8350,8 +9349,95 @@ static int sp_3072_mod_exp_26(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 52); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 52, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 52); + } + rt = td + (17 * 52); + sq = td + (18 * 52); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_26(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_26(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_26(t[0], t[0], norm); + err = sp_3072_mod_26(t[0], t[0], m); + } + } + else { + sp_3072_mul_26(t[0], a, norm); + err = sp_3072_mod_26(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_26(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_3072_mont_mul_26(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 52); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 60] >> (i % 60)) & 1) == 0) { + sp_3072_mont_sqr_26(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 60] >> (l % 60)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 60] >> (j % 60)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_26(rt, rt, m, mp); + } + sp_3072_mont_mul_26(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_26(rt, m, mp); + n = sp_3072_cmp_26(rt, m); + sp_3072_cond_sub_26(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 52); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 52); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -8427,9 +9513,10 @@ static int sp_3072_mod_exp_26(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (32 * 52) + 52); - sp_digit* t[32]; + SP_DECL_VAR(sp_digit, td, (16 * 52) + 104); + sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -8442,13 +9529,14 @@ static int sp_3072_mod_exp_26(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (32 * 52) + 52, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 52) + 104, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<32; i++) + for (i=0; i<16; i++) t[i] = td + i * 52; - rt = td + 1664; + rt = td + 832; + ct = td + 884; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_26(norm, m); @@ -8481,24 +9569,8 @@ static int sp_3072_mod_exp_26(sp_digit* r, const sp_digit* a, sp_3072_mont_mul_26(t[13], t[ 7], t[ 6], m, mp); sp_3072_mont_sqr_26(t[14], t[ 7], m, mp); sp_3072_mont_mul_26(t[15], t[ 8], t[ 7], m, mp); - sp_3072_mont_sqr_26(t[16], t[ 8], m, mp); - sp_3072_mont_mul_26(t[17], t[ 9], t[ 8], m, mp); - sp_3072_mont_sqr_26(t[18], t[ 9], m, mp); - sp_3072_mont_mul_26(t[19], t[10], t[ 9], m, mp); - sp_3072_mont_sqr_26(t[20], t[10], m, mp); - sp_3072_mont_mul_26(t[21], t[11], t[10], m, mp); - sp_3072_mont_sqr_26(t[22], t[11], m, mp); - sp_3072_mont_mul_26(t[23], t[12], t[11], m, mp); - sp_3072_mont_sqr_26(t[24], t[12], m, mp); - sp_3072_mont_mul_26(t[25], t[13], t[12], m, mp); - sp_3072_mont_sqr_26(t[26], t[13], m, mp); - sp_3072_mont_mul_26(t[27], t[14], t[13], m, mp); - sp_3072_mont_sqr_26(t[28], t[14], m, mp); - sp_3072_mont_mul_26(t[29], t[15], t[14], m, mp); - sp_3072_mont_sqr_26(t[30], t[15], m, mp); - sp_3072_mont_mul_26(t[31], t[16], t[15], m, mp); - bits = ((bits + 4) / 5) * 5; + bits = ((bits + 3) / 4) * 4; i = ((bits + 59) / 60) - 1; c = bits % 60; if (c == 0) { @@ -8511,19 +9583,152 @@ static int sp_3072_mod_exp_26(sp_digit* r, const sp_digit* a, n = 0; i--; } - if (c < 5) { + if (c < 4) { n |= (sp_uint64)e[i--] << (4 - c); c += 60; } - y = (int)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 52); - while ((i >= 0) || (c >= 5)) { - if (c >= 5) { - y = (byte)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_3072_get_from_table_26(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 4; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 56; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 4; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 60 - c; + } + + sp_3072_mont_sqr_26(rt, rt, m, mp); + sp_3072_mont_sqr_26(rt, rt, m, mp); + sp_3072_mont_sqr_26(rt, rt, m, mp); + sp_3072_mont_sqr_26(rt, rt, m, mp); + + sp_3072_get_from_table_26(ct, t, y); + sp_3072_mont_mul_26(rt, rt, ct, m, mp); + } + + sp_3072_mont_reduce_26(rt, m, mp); + n = sp_3072_cmp_26(rt, m); + sp_3072_cond_sub_26(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 52); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +#else + SP_DECL_VAR(sp_digit, td, (32 * 52) + 52); + sp_digit* t[32]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (32 * 52) + 52, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<32; i++) + t[i] = td + i * 52; + rt = td + 1664; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_26(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_26(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_26(t[1], t[1], norm); + err = sp_3072_mod_26(t[1], t[1], m); + } + } + else { + sp_3072_mul_26(t[1], a, norm); + err = sp_3072_mod_26(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_26(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_26(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_26(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_26(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_26(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_26(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_26(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_26(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_26(t[10], t[ 5], m, mp); + sp_3072_mont_mul_26(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_26(t[12], t[ 6], m, mp); + sp_3072_mont_mul_26(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_26(t[14], t[ 7], m, mp); + sp_3072_mont_mul_26(t[15], t[ 8], t[ 7], m, mp); + sp_3072_mont_sqr_26(t[16], t[ 8], m, mp); + sp_3072_mont_mul_26(t[17], t[ 9], t[ 8], m, mp); + sp_3072_mont_sqr_26(t[18], t[ 9], m, mp); + sp_3072_mont_mul_26(t[19], t[10], t[ 9], m, mp); + sp_3072_mont_sqr_26(t[20], t[10], m, mp); + sp_3072_mont_mul_26(t[21], t[11], t[10], m, mp); + sp_3072_mont_sqr_26(t[22], t[11], m, mp); + sp_3072_mont_mul_26(t[23], t[12], t[11], m, mp); + sp_3072_mont_sqr_26(t[24], t[12], m, mp); + sp_3072_mont_mul_26(t[25], t[13], t[12], m, mp); + sp_3072_mont_sqr_26(t[26], t[13], m, mp); + sp_3072_mont_mul_26(t[27], t[14], t[13], m, mp); + sp_3072_mont_sqr_26(t[28], t[14], m, mp); + sp_3072_mont_mul_26(t[29], t[15], t[14], m, mp); + sp_3072_mont_sqr_26(t[30], t[15], m, mp); + sp_3072_mont_mul_26(t[31], t[16], t[15], m, mp); + + bits = ((bits + 4) / 5) * 5; + i = ((bits + 59) / 60) - 1; + c = bits % 60; + if (c == 0) { + c = 60; + } + if (i < 26) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 5) { + n |= (sp_uint64)e[i--] << (4 - c); + c += 60; + } + y = (int)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 52); + while ((i >= 0) || (c >= 5)) { + if (c >= 5) { + y = (byte)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; } else if (c == 0) { n = (sp_uint64)e[i--] << 4; @@ -9062,6 +10267,40 @@ static int sp_3072_mod_52(sp_digit* r, const sp_digit* a, const sp_digit* m) } #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_52(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 104; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 104; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -9154,8 +10393,95 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 104); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 104, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 104); + } + rt = td + (9 * 104); + sq = td + (10 * 104); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_52(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_52(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_52(t[0], t[0], norm); + err = sp_3072_mod_52(t[0], t[0], m); + } + } + else { + sp_3072_mul_52(t[0], a, norm); + err = sp_3072_mod_52(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_52(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_3072_mont_mul_52(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 104); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 60] >> (i % 60)) & 1) == 0) { + sp_3072_mont_sqr_52(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 60] >> (l % 60)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 60] >> (j % 60)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_52(rt, rt, m, mp); + } + sp_3072_mont_mul_52(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_52(rt, m, mp); + n = sp_3072_cmp_52(rt, m); + sp_3072_cond_sub_52(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 104); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 104); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -9231,9 +10557,10 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (16 * 104) + 104); + SP_DECL_VAR(sp_digit, td, (16 * 104) + 208); sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -9246,13 +10573,14 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (16 * 104) + 104, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 104) + 208, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; for (i=0; i<16; i++) t[i] = td + i * 104; rt = td + 1664; + ct = td + 1768; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_52(norm, m); @@ -9306,7 +10634,7 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, y = (int)((n >> 60) & 0xf); n = (sp_uint64)n << 4; c -= 4; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 104); + sp_3072_get_from_table_52(rt, t, y); while ((i >= 0) || (c >= 4)) { if (c >= 4) { y = (byte)((n >> 60) & 0xf); @@ -9333,7 +10661,8 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, sp_3072_mont_sqr_52(rt, rt, m, mp); sp_3072_mont_sqr_52(rt, rt, m, mp); - sp_3072_mont_mul_52(rt, rt, t[y], m, mp); + sp_3072_get_from_table_52(ct, t, y); + sp_3072_mont_mul_52(rt, rt, ct, m, mp); } sp_3072_mont_reduce_52(rt, m, mp); @@ -9346,60 +10675,176 @@ static int sp_3072_mod_exp_52(sp_digit* r, const sp_digit* a, return err; #endif -} -#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ - !defined(WOLFSSL_SP_FAST_MODEXP) -/* Non-blocking modular exponentiation. State machine driven by sp_ctx; - * each call advances one Montgomery op or one bit-extract step then - * returns MP_WOULDBLOCK until the final reduction completes. - */ -typedef struct sp_3072_mod_exp_52_ctx { - int state; - sp_digit td[3 * 104]; - sp_digit* t[3]; - sp_digit* norm; - sp_digit mp; +#else + SP_DECL_VAR(sp_digit, td, (16 * 104) + 104); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; sp_digit n; int i; int c; byte y; - int reduceA; - int bits; -} sp_3072_mod_exp_52_ctx; + int err = MP_OKAY; -/* Modular exponentiate a to the e mod m. (r = a^e mod m) - * - * Non-blocking version. Call repeatedly with the same context until it does - * not return MP_WOULDBLOCK. State is saved and restored through ctx. - * - * @param [in, out] ctx Context saving state for the non-blocking - * operation. - * @param [out] r A single precision number that is the result of the - * operation. - * @param [in] a A single precision number being exponentiated. - * @param [in] e A single precision number that is the exponent. - * @param [in] bits The number of bits in the exponent. - * @param [in] m A single precision number that is the modulus. - * @param [in] reduceA Whether to reduce a modulo m before the operation. - * - * @return MP_OKAY on completion. - * @return MP_WOULDBLOCK while more work remains. - */ -static int sp_3072_mod_exp_52_nb(sp_3072_mod_exp_52_ctx* ctx, - sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, - const sp_digit* m, int reduceA) -{ - int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); - int j; + if (bits == 0) { + err = MP_VAL; + } - switch (ctx->state) { - case 0: /* INIT: layout td, mont_setup, mont_norm */ - if (bits == 0) { - err = MP_VAL; - break; - } - for (j = 0; j < 3; j++) { - ctx->t[j] = ctx->td + (j * 52 * 2); + SP_ALLOC_VAR(sp_digit, td, (16 * 104) + 104, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 104; + rt = td + 1664; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_52(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_52(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_52(t[1], t[1], norm); + err = sp_3072_mod_52(t[1], t[1], m); + } + } + else { + sp_3072_mul_52(t[1], a, norm); + err = sp_3072_mod_52(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_52(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_52(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_52(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_52(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_52(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_52(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_52(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_52(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_52(t[10], t[ 5], m, mp); + sp_3072_mont_mul_52(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_52(t[12], t[ 6], m, mp); + sp_3072_mont_mul_52(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_52(t[14], t[ 7], m, mp); + sp_3072_mont_mul_52(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 59) / 60) - 1; + c = bits % 60; + if (c == 0) { + c = 60; + } + if (i < 52) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (4 - c); + c += 60; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 104); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 4; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 56; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 4; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 60 - c; + } + + sp_3072_mont_sqr_52(rt, rt, m, mp); + sp_3072_mont_sqr_52(rt, rt, m, mp); + sp_3072_mont_sqr_52(rt, rt, m, mp); + sp_3072_mont_sqr_52(rt, rt, m, mp); + + sp_3072_mont_mul_52(rt, rt, t[y], m, mp); + } + + sp_3072_mont_reduce_52(rt, m, mp); + n = sp_3072_cmp_52(rt, m); + sp_3072_cond_sub_52(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 104); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} +#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ + !defined(WOLFSSL_SP_FAST_MODEXP) +/* Non-blocking modular exponentiation. State machine driven by sp_ctx; + * each call advances one Montgomery op or one bit-extract step then + * returns MP_WOULDBLOCK until the final reduction completes. + */ +typedef struct sp_3072_mod_exp_52_ctx { + int state; + sp_digit td[3 * 104]; + sp_digit* t[3]; + sp_digit* norm; + sp_digit mp; + sp_digit n; + int i; + int c; + byte y; + int reduceA; + int bits; +} sp_3072_mod_exp_52_ctx; + +/* Modular exponentiate a to the e mod m. (r = a^e mod m) + * + * Non-blocking version. Call repeatedly with the same context until it does + * not return MP_WOULDBLOCK. State is saved and restored through ctx. + * + * @param [in, out] ctx Context saving state for the non-blocking + * operation. + * @param [out] r A single precision number that is the result of the + * operation. + * @param [in] a A single precision number being exponentiated. + * @param [in] e A single precision number that is the exponent. + * @param [in] bits The number of bits in the exponent. + * @param [in] m A single precision number that is the modulus. + * @param [in] reduceA Whether to reduce a modulo m before the operation. + * + * @return MP_OKAY on completion. + * @return MP_WOULDBLOCK while more work remains. + */ +static int sp_3072_mod_exp_52_nb(sp_3072_mod_exp_52_ctx* ctx, + sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, + const sp_digit* m, int reduceA) +{ + int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); + int j; + + switch (ctx->state) { + case 0: /* INIT: layout td, mont_setup, mont_norm */ + if (bits == 0) { + err = MP_VAL; + break; + } + for (j = 0; j < 3; j++) { + ctx->t[j] = ctx->td + (j * 52 * 2); XMEMSET(ctx->t[j], 0, sizeof(sp_digit) * 52U * 2U); } ctx->norm = ctx->t[0]; @@ -12253,6 +13698,40 @@ static int sp_3072_mod_27(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_3072_div_27(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_27(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 54; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 54; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -12345,8 +13824,95 @@ static int sp_3072_mod_exp_27(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 54); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 54, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 54); + } + rt = td + (17 * 54); + sq = td + (18 * 54); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_27(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_27(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_27(t[0], t[0], norm); + err = sp_3072_mod_27(t[0], t[0], m); + } + } + else { + sp_3072_mul_27(t[0], a, norm); + err = sp_3072_mod_27(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_27(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_3072_mont_mul_27(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 54); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 57] >> (i % 57)) & 1) == 0) { + sp_3072_mont_sqr_27(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 57] >> (l % 57)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 57] >> (j % 57)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_27(rt, rt, m, mp); + } + sp_3072_mont_mul_27(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_27(rt, m, mp); + n = sp_3072_cmp_27(rt, m); + sp_3072_cond_sub_27(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 54); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 54); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -12422,9 +13988,10 @@ static int sp_3072_mod_exp_27(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (32 * 54) + 54); - sp_digit* t[32]; + SP_DECL_VAR(sp_digit, td, (16 * 54) + 108); + sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -12437,13 +14004,14 @@ static int sp_3072_mod_exp_27(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (32 * 54) + 54, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 54) + 108, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; - for (i=0; i<32; i++) + for (i=0; i<16; i++) t[i] = td + i * 54; - rt = td + 1728; + rt = td + 864; + ct = td + 918; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_27(norm, m); @@ -12476,24 +14044,8 @@ static int sp_3072_mod_exp_27(sp_digit* r, const sp_digit* a, sp_3072_mont_mul_27(t[13], t[ 7], t[ 6], m, mp); sp_3072_mont_sqr_27(t[14], t[ 7], m, mp); sp_3072_mont_mul_27(t[15], t[ 8], t[ 7], m, mp); - sp_3072_mont_sqr_27(t[16], t[ 8], m, mp); - sp_3072_mont_mul_27(t[17], t[ 9], t[ 8], m, mp); - sp_3072_mont_sqr_27(t[18], t[ 9], m, mp); - sp_3072_mont_mul_27(t[19], t[10], t[ 9], m, mp); - sp_3072_mont_sqr_27(t[20], t[10], m, mp); - sp_3072_mont_mul_27(t[21], t[11], t[10], m, mp); - sp_3072_mont_sqr_27(t[22], t[11], m, mp); - sp_3072_mont_mul_27(t[23], t[12], t[11], m, mp); - sp_3072_mont_sqr_27(t[24], t[12], m, mp); - sp_3072_mont_mul_27(t[25], t[13], t[12], m, mp); - sp_3072_mont_sqr_27(t[26], t[13], m, mp); - sp_3072_mont_mul_27(t[27], t[14], t[13], m, mp); - sp_3072_mont_sqr_27(t[28], t[14], m, mp); - sp_3072_mont_mul_27(t[29], t[15], t[14], m, mp); - sp_3072_mont_sqr_27(t[30], t[15], m, mp); - sp_3072_mont_mul_27(t[31], t[16], t[15], m, mp); - bits = ((bits + 4) / 5) * 5; + bits = ((bits + 3) / 4) * 4; i = ((bits + 56) / 57) - 1; c = bits % 57; if (c == 0) { @@ -12506,19 +14058,152 @@ static int sp_3072_mod_exp_27(sp_digit* r, const sp_digit* a, n = 0; i--; } - if (c < 5) { + if (c < 4) { n |= (sp_uint64)e[i--] << (7 - c); c += 57; } - y = (int)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 54); - while ((i >= 0) || (c >= 5)) { - if (c >= 5) { - y = (byte)((n >> 59) & 0x1f); - n = (sp_uint64)n << 5; - c -= 5; + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_3072_get_from_table_27(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 7; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 53; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 7; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 57 - c; + } + + sp_3072_mont_sqr_27(rt, rt, m, mp); + sp_3072_mont_sqr_27(rt, rt, m, mp); + sp_3072_mont_sqr_27(rt, rt, m, mp); + sp_3072_mont_sqr_27(rt, rt, m, mp); + + sp_3072_get_from_table_27(ct, t, y); + sp_3072_mont_mul_27(rt, rt, ct, m, mp); + } + + sp_3072_mont_reduce_27(rt, m, mp); + n = sp_3072_cmp_27(rt, m); + sp_3072_cond_sub_27(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 54); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +#else + SP_DECL_VAR(sp_digit, td, (32 * 54) + 54); + sp_digit* t[32]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (32 * 54) + 54, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<32; i++) + t[i] = td + i * 54; + rt = td + 1728; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_27(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_27(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_27(t[1], t[1], norm); + err = sp_3072_mod_27(t[1], t[1], m); + } + } + else { + sp_3072_mul_27(t[1], a, norm); + err = sp_3072_mod_27(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_27(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_27(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_27(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_27(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_27(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_27(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_27(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_27(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_27(t[10], t[ 5], m, mp); + sp_3072_mont_mul_27(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_27(t[12], t[ 6], m, mp); + sp_3072_mont_mul_27(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_27(t[14], t[ 7], m, mp); + sp_3072_mont_mul_27(t[15], t[ 8], t[ 7], m, mp); + sp_3072_mont_sqr_27(t[16], t[ 8], m, mp); + sp_3072_mont_mul_27(t[17], t[ 9], t[ 8], m, mp); + sp_3072_mont_sqr_27(t[18], t[ 9], m, mp); + sp_3072_mont_mul_27(t[19], t[10], t[ 9], m, mp); + sp_3072_mont_sqr_27(t[20], t[10], m, mp); + sp_3072_mont_mul_27(t[21], t[11], t[10], m, mp); + sp_3072_mont_sqr_27(t[22], t[11], m, mp); + sp_3072_mont_mul_27(t[23], t[12], t[11], m, mp); + sp_3072_mont_sqr_27(t[24], t[12], m, mp); + sp_3072_mont_mul_27(t[25], t[13], t[12], m, mp); + sp_3072_mont_sqr_27(t[26], t[13], m, mp); + sp_3072_mont_mul_27(t[27], t[14], t[13], m, mp); + sp_3072_mont_sqr_27(t[28], t[14], m, mp); + sp_3072_mont_mul_27(t[29], t[15], t[14], m, mp); + sp_3072_mont_sqr_27(t[30], t[15], m, mp); + sp_3072_mont_mul_27(t[31], t[16], t[15], m, mp); + + bits = ((bits + 4) / 5) * 5; + i = ((bits + 56) / 57) - 1; + c = bits % 57; + if (c == 0) { + c = 57; + } + if (i < 27) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 5) { + n |= (sp_uint64)e[i--] << (7 - c); + c += 57; + } + y = (int)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 54); + while ((i >= 0) || (c >= 5)) { + if (c >= 5) { + y = (byte)((n >> 59) & 0x1f); + n = (sp_uint64)n << 5; + c -= 5; } else if (c == 0) { n = (sp_uint64)e[i--] << 7; @@ -13125,6 +14810,40 @@ static int sp_3072_mod_54(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_3072_get_from_table_54(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 108; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 108; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -13217,8 +14936,95 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 108); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 108, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 108); + } + rt = td + (9 * 108); + sq = td + (10 * 108); + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_54(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_54(t[0], a, m); + if (err == MP_OKAY) { + sp_3072_mul_54(t[0], t[0], norm); + err = sp_3072_mod_54(t[0], t[0], m); + } + } + else { + sp_3072_mul_54(t[0], a, norm); + err = sp_3072_mod_54(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_3072_mont_sqr_54(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_3072_mont_mul_54(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 108); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 57] >> (i % 57)) & 1) == 0) { + sp_3072_mont_sqr_54(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 57] >> (l % 57)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 57] >> (j % 57)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_3072_mont_sqr_54(rt, rt, m, mp); + } + sp_3072_mont_mul_54(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_3072_mont_reduce_54(rt, m, mp); + n = sp_3072_cmp_54(rt, m); + sp_3072_cond_sub_54(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 108); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 108); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -13294,9 +15100,10 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, return err; #else - SP_DECL_VAR(sp_digit, td, (16 * 108) + 108); + SP_DECL_VAR(sp_digit, td, (16 * 108) + 216); sp_digit* t[16]; sp_digit* rt = NULL; + sp_digit* ct = NULL; sp_digit* norm = NULL; sp_digit mp = 1; sp_digit n; @@ -13309,13 +15116,14 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, err = MP_VAL; } - SP_ALLOC_VAR(sp_digit, td, (16 * 108) + 108, NULL, + SP_ALLOC_VAR(sp_digit, td, (16 * 108) + 216, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (err == MP_OKAY) { norm = td; for (i=0; i<16; i++) t[i] = td + i * 108; rt = td + 1728; + ct = td + 1836; sp_3072_mont_setup(m, &mp); sp_3072_mont_norm_54(norm, m); @@ -13369,7 +15177,7 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, y = (int)((n >> 60) & 0xf); n = (sp_uint64)n << 4; c -= 4; - XMEMCPY(rt, t[y], sizeof(sp_digit) * 108); + sp_3072_get_from_table_54(rt, t, y); while ((i >= 0) || (c >= 4)) { if (c >= 4) { y = (byte)((n >> 60) & 0xf); @@ -13396,7 +15204,8 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, sp_3072_mont_sqr_54(rt, rt, m, mp); sp_3072_mont_sqr_54(rt, rt, m, mp); - sp_3072_mont_mul_54(rt, rt, t[y], m, mp); + sp_3072_get_from_table_54(ct, t, y); + sp_3072_mont_mul_54(rt, rt, ct, m, mp); } sp_3072_mont_reduce_54(rt, m, mp); @@ -13409,51 +15218,167 @@ static int sp_3072_mod_exp_54(sp_digit* r, const sp_digit* a, return err; #endif -} -#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ - !defined(WOLFSSL_SP_FAST_MODEXP) -/* Non-blocking modular exponentiation. State machine driven by sp_ctx; - * each call advances one Montgomery op or one bit-extract step then - * returns MP_WOULDBLOCK until the final reduction completes. - */ -typedef struct sp_3072_mod_exp_54_ctx { - int state; - sp_digit td[3 * 108]; - sp_digit* t[3]; - sp_digit* norm; - sp_digit mp; +#else + SP_DECL_VAR(sp_digit, td, (16 * 108) + 108); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; sp_digit n; int i; int c; byte y; - int reduceA; - int bits; -} sp_3072_mod_exp_54_ctx; + int err = MP_OKAY; -/* Modular exponentiate a to the e mod m. (r = a^e mod m) - * - * Non-blocking version. Call repeatedly with the same context until it does - * not return MP_WOULDBLOCK. State is saved and restored through ctx. - * - * @param [in, out] ctx Context saving state for the non-blocking - * operation. - * @param [out] r A single precision number that is the result of the - * operation. - * @param [in] a A single precision number being exponentiated. - * @param [in] e A single precision number that is the exponent. - * @param [in] bits The number of bits in the exponent. - * @param [in] m A single precision number that is the modulus. - * @param [in] reduceA Whether to reduce a modulo m before the operation. - * - * @return MP_OKAY on completion. - * @return MP_WOULDBLOCK while more work remains. - */ -static int sp_3072_mod_exp_54_nb(sp_3072_mod_exp_54_ctx* ctx, - sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, - const sp_digit* m, int reduceA) -{ - int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); - int j; + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 108) + 108, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 108; + rt = td + 1728; + + sp_3072_mont_setup(m, &mp); + sp_3072_mont_norm_54(norm, m); + + if (reduceA != 0) { + err = sp_3072_mod_54(t[1], a, m); + if (err == MP_OKAY) { + sp_3072_mul_54(t[1], t[1], norm); + err = sp_3072_mod_54(t[1], t[1], m); + } + } + else { + sp_3072_mul_54(t[1], a, norm); + err = sp_3072_mod_54(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_3072_mont_sqr_54(t[ 2], t[ 1], m, mp); + sp_3072_mont_mul_54(t[ 3], t[ 2], t[ 1], m, mp); + sp_3072_mont_sqr_54(t[ 4], t[ 2], m, mp); + sp_3072_mont_mul_54(t[ 5], t[ 3], t[ 2], m, mp); + sp_3072_mont_sqr_54(t[ 6], t[ 3], m, mp); + sp_3072_mont_mul_54(t[ 7], t[ 4], t[ 3], m, mp); + sp_3072_mont_sqr_54(t[ 8], t[ 4], m, mp); + sp_3072_mont_mul_54(t[ 9], t[ 5], t[ 4], m, mp); + sp_3072_mont_sqr_54(t[10], t[ 5], m, mp); + sp_3072_mont_mul_54(t[11], t[ 6], t[ 5], m, mp); + sp_3072_mont_sqr_54(t[12], t[ 6], m, mp); + sp_3072_mont_mul_54(t[13], t[ 7], t[ 6], m, mp); + sp_3072_mont_sqr_54(t[14], t[ 7], m, mp); + sp_3072_mont_mul_54(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 56) / 57) - 1; + c = bits % 57; + if (c == 0) { + c = 57; + } + if (i < 54) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (7 - c); + c += 57; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + XMEMCPY(rt, t[y], sizeof(sp_digit) * 108); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 7; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 53; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 7; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 57 - c; + } + + sp_3072_mont_sqr_54(rt, rt, m, mp); + sp_3072_mont_sqr_54(rt, rt, m, mp); + sp_3072_mont_sqr_54(rt, rt, m, mp); + sp_3072_mont_sqr_54(rt, rt, m, mp); + + sp_3072_mont_mul_54(rt, rt, t[y], m, mp); + } + + sp_3072_mont_reduce_54(rt, m, mp); + n = sp_3072_cmp_54(rt, m); + sp_3072_cond_sub_54(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 108); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif +} +#if defined(WOLFSSL_SP_NONBLOCK) && defined(WOLFSSL_SP_SMALL) && \ + !defined(WOLFSSL_SP_FAST_MODEXP) +/* Non-blocking modular exponentiation. State machine driven by sp_ctx; + * each call advances one Montgomery op or one bit-extract step then + * returns MP_WOULDBLOCK until the final reduction completes. + */ +typedef struct sp_3072_mod_exp_54_ctx { + int state; + sp_digit td[3 * 108]; + sp_digit* t[3]; + sp_digit* norm; + sp_digit mp; + sp_digit n; + int i; + int c; + byte y; + int reduceA; + int bits; +} sp_3072_mod_exp_54_ctx; + +/* Modular exponentiate a to the e mod m. (r = a^e mod m) + * + * Non-blocking version. Call repeatedly with the same context until it does + * not return MP_WOULDBLOCK. State is saved and restored through ctx. + * + * @param [in, out] ctx Context saving state for the non-blocking + * operation. + * @param [out] r A single precision number that is the result of the + * operation. + * @param [in] a A single precision number being exponentiated. + * @param [in] e A single precision number that is the exponent. + * @param [in] bits The number of bits in the exponent. + * @param [in] m A single precision number that is the modulus. + * @param [in] reduceA Whether to reduce a modulo m before the operation. + * + * @return MP_OKAY on completion. + * @return MP_WOULDBLOCK while more work remains. + */ +static int sp_3072_mod_exp_54_nb(sp_3072_mod_exp_54_ctx* ctx, + sp_digit* r, const sp_digit* a, const sp_digit* e, int bits, + const sp_digit* m, int reduceA) +{ + int err = WC_NO_ERR_TRACE(MP_WOULDBLOCK); + int j; switch (ctx->state) { case 0: /* INIT: layout td, mont_setup, mont_norm */ @@ -15555,6 +17480,40 @@ static int sp_4096_mod_35(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_4096_div_35(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_35(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 70; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 70; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -15647,8 +17606,95 @@ static int sp_4096_mod_exp_35(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 70); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 70, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 70); + } + rt = td + (17 * 70); + sq = td + (18 * 70); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_35(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_35(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_35(t[0], t[0], norm); + err = sp_4096_mod_35(t[0], t[0], m); + } + } + else { + sp_4096_mul_35(t[0], a, norm); + err = sp_4096_mod_35(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_35(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_4096_mont_mul_35(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 70); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 59] >> (i % 59)) & 1) == 0) { + sp_4096_mont_sqr_35(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 59] >> (l % 59)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 59] >> (j % 59)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_35(rt, rt, m, mp); + } + sp_4096_mont_mul_35(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_35(rt, m, mp); + n = sp_4096_cmp_35(rt, m); + sp_4096_cond_sub_35(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 70); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 70); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -15723,6 +17769,125 @@ static int sp_4096_mod_exp_35(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 70) + 140); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 70) + 140, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 70; + rt = td + 1120; + ct = td + 1190; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_35(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_35(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_35(t[1], t[1], norm); + err = sp_4096_mod_35(t[1], t[1], m); + } + } + else { + sp_4096_mul_35(t[1], a, norm); + err = sp_4096_mod_35(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_35(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_35(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_35(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_35(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_35(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_35(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_35(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_35(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_35(t[10], t[ 5], m, mp); + sp_4096_mont_mul_35(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_35(t[12], t[ 6], m, mp); + sp_4096_mont_mul_35(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_35(t[14], t[ 7], m, mp); + sp_4096_mont_mul_35(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 58) / 59) - 1; + c = bits % 59; + if (c == 0) { + c = 59; + } + if (i < 35) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (5 - c); + c += 59; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_4096_get_from_table_35(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 5; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 55; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 5; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 59 - c; + } + + sp_4096_mont_sqr_35(rt, rt, m, mp); + sp_4096_mont_sqr_35(rt, rt, m, mp); + sp_4096_mont_sqr_35(rt, rt, m, mp); + sp_4096_mont_sqr_35(rt, rt, m, mp); + + sp_4096_get_from_table_35(ct, t, y); + sp_4096_mont_mul_35(rt, rt, ct, m, mp); + } + + sp_4096_mont_reduce_35(rt, m, mp); + n = sp_4096_cmp_35(rt, m); + sp_4096_cond_sub_35(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 70); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 70) + 70); sp_digit* t[32]; @@ -16354,6 +18519,40 @@ static int sp_4096_mod_70(sp_digit* r, const sp_digit* a, const sp_digit* m) } #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_70(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 140; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 140; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -16446,8 +18645,95 @@ static int sp_4096_mod_exp_70(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 140); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 140, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 140); + } + rt = td + (9 * 140); + sq = td + (10 * 140); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_70(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_70(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_70(t[0], t[0], norm); + err = sp_4096_mod_70(t[0], t[0], m); + } + } + else { + sp_4096_mul_70(t[0], a, norm); + err = sp_4096_mod_70(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_70(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_4096_mont_mul_70(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 140); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 59] >> (i % 59)) & 1) == 0) { + sp_4096_mont_sqr_70(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 59] >> (l % 59)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 59] >> (j % 59)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_70(rt, rt, m, mp); + } + sp_4096_mont_mul_70(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_70(rt, m, mp); + n = sp_4096_cmp_70(rt, m); + sp_4096_cond_sub_70(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 140); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 140); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -16480,48 +18766,167 @@ static int sp_4096_mod_exp_70(sp_digit* r, const sp_digit* a, } } else { - sp_4096_mul_70(t[1], a, norm); - err = sp_4096_mod_70(t[1], t[1], m); + sp_4096_mul_70(t[1], a, norm); + err = sp_4096_mod_70(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + i = (bits - 1) / 59; + c = ((bits - 1) % 59) + 1; + n = (sp_uint64)e[i--] << (59 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 59; + } + + y = (int)((n >> 58) & 1); + n = (sp_uint64)n << 1; + + sp_4096_mont_mul_70(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 70 * 2); + sp_4096_mont_sqr_70(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 70 * 2); + } + + sp_4096_mont_reduce_70(t[0], m, mp); + n = sp_4096_cmp_70(t[0], m); + sp_4096_cond_sub_70(t[0], t[0], m, (sp_digit)~(n >> 63)); + XMEMCPY(r, t[0], sizeof(*r) * 70 * 2); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 140) + 280); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 140) + 280, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 140; + rt = td + 2240; + ct = td + 2380; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_70(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_70(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_70(t[1], t[1], norm); + err = sp_4096_mod_70(t[1], t[1], m); + } + } + else { + sp_4096_mul_70(t[1], a, norm); + err = sp_4096_mod_70(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_70(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_70(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_70(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_70(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_70(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_70(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_70(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_70(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_70(t[10], t[ 5], m, mp); + sp_4096_mont_mul_70(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_70(t[12], t[ 6], m, mp); + sp_4096_mont_mul_70(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_70(t[14], t[ 7], m, mp); + sp_4096_mont_mul_70(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 58) / 59) - 1; + c = bits % 59; + if (c == 0) { + c = 59; + } + if (i < 70) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; } - } - - if (err == MP_OKAY) { - i = (bits - 1) / 59; - c = ((bits - 1) % 59) + 1; - n = (sp_uint64)e[i--] << (59 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } - - n = e[i--]; - c = 59; + if (c < 4) { + n |= (sp_uint64)e[i--] << (5 - c); + c += 59; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_4096_get_from_table_70(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 5; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 55; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 5; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 59 - c; } - y = (int)((n >> 58) & 1); - n = (sp_uint64)n << 1; - - sp_4096_mont_mul_70(t[y^1], t[0], t[1], m, mp); + sp_4096_mont_sqr_70(rt, rt, m, mp); + sp_4096_mont_sqr_70(rt, rt, m, mp); + sp_4096_mont_sqr_70(rt, rt, m, mp); + sp_4096_mont_sqr_70(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 70 * 2); - sp_4096_mont_sqr_70(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 70 * 2); + sp_4096_get_from_table_70(ct, t, y); + sp_4096_mont_mul_70(rt, rt, ct, m, mp); } - sp_4096_mont_reduce_70(t[0], m, mp); - n = sp_4096_cmp_70(t[0], m); - sp_4096_cond_sub_70(t[0], t[0], m, (sp_digit)~(n >> 63)); - XMEMCPY(r, t[0], sizeof(*r) * 70 * 2); + sp_4096_mont_reduce_70(rt, m, mp); + n = sp_4096_cmp_70(rt, m); + sp_4096_cond_sub_70(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 140); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 140) + 140); sp_digit* t[16]; @@ -19643,6 +22048,40 @@ static int sp_4096_mod_39(sp_digit* r, const sp_digit* a, const sp_digit* m) return sp_4096_div_39(a, m, NULL, r); } +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_39(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 78; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 78; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -19735,8 +22174,95 @@ static int sp_4096_mod_exp_39(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 19 * 78); + sp_digit* t[16]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 19 * 78, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) { + t[i] = td + ((i + 1) * 78); + } + rt = td + (17 * 78); + sq = td + (18 * 78); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_39(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_39(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_39(t[0], t[0], norm); + err = sp_4096_mod_39(t[0], t[0], m); + } + } + else { + sp_4096_mul_39(t[0], a, norm); + err = sp_4096_mod_39(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_39(sq, t[0], m, mp); + for (i = 1; i < 16; i++) { + sp_4096_mont_mul_39(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 78); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 53] >> (i % 53)) & 1) == 0) { + sp_4096_mont_sqr_39(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 5 bits ending in a set bit. */ + l = (i >= 5) ? (i - 5 + 1) : 0; + while (((e[l / 53] >> (l % 53)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 53] >> (j % 53)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_39(rt, rt, m, mp); + } + sp_4096_mont_mul_39(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_39(rt, m, mp); + n = sp_4096_cmp_39(rt, m); + sp_4096_cond_sub_39(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 78); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 78); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -19757,60 +22283,179 @@ static int sp_4096_mod_exp_39(sp_digit* r, const sp_digit* a, for (i=0; i<3; i++) { t[i] = td + (i * 39 * 2); } - - sp_4096_mont_setup(m, &mp); - sp_4096_mont_norm_39(norm, m); - - if (reduceA != 0) { - err = sp_4096_mod_39(t[1], a, m); - if (err == MP_OKAY) { - sp_4096_mul_39(t[1], t[1], norm); - err = sp_4096_mod_39(t[1], t[1], m); - } + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_39(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_39(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_39(t[1], t[1], norm); + err = sp_4096_mod_39(t[1], t[1], m); + } + } + else { + sp_4096_mul_39(t[1], a, norm); + err = sp_4096_mod_39(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + i = (bits - 1) / 53; + c = ((bits - 1) % 53) + 1; + n = (sp_uint64)e[i--] << (53 - c); + for (; ; c--) { + if (c == 0) { + if (i == -1) { + break; + } + + n = e[i--]; + c = 53; + } + + y = (int)((n >> 52) & 1); + n = (sp_uint64)n << 1; + + sp_4096_mont_mul_39(t[y^1], t[0], t[1], m, mp); + + XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), + sizeof(*t[2]) * 39 * 2); + sp_4096_mont_sqr_39(t[2], t[2], m, mp); + XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + + ((size_t)t[1] & addr_mask[y])), t[2], + sizeof(*t[2]) * 39 * 2); + } + + sp_4096_mont_reduce_39(t[0], m, mp); + n = sp_4096_cmp_39(t[0], m); + sp_4096_cond_sub_39(t[0], t[0], m, (sp_digit)~(n >> 63)); + XMEMCPY(r, t[0], sizeof(*r) * 39 * 2); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 78) + 156); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 78) + 156, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 78; + rt = td + 1248; + ct = td + 1326; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_39(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_39(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_39(t[1], t[1], norm); + err = sp_4096_mod_39(t[1], t[1], m); + } + } + else { + sp_4096_mul_39(t[1], a, norm); + err = sp_4096_mod_39(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_39(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_39(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_39(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_39(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_39(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_39(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_39(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_39(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_39(t[10], t[ 5], m, mp); + sp_4096_mont_mul_39(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_39(t[12], t[ 6], m, mp); + sp_4096_mont_mul_39(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_39(t[14], t[ 7], m, mp); + sp_4096_mont_mul_39(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 52) / 53) - 1; + c = bits % 53; + if (c == 0) { + c = 53; + } + if (i < 39) { + n = (sp_uint64)e[i--] << (64 - c); } else { - sp_4096_mul_39(t[1], a, norm); - err = sp_4096_mod_39(t[1], t[1], m); + n = 0; + i--; } - } - - if (err == MP_OKAY) { - i = (bits - 1) / 53; - c = ((bits - 1) % 53) + 1; - n = (sp_uint64)e[i--] << (53 - c); - for (; ; c--) { - if (c == 0) { - if (i == -1) { - break; - } - - n = e[i--]; - c = 53; + if (c < 4) { + n |= (sp_uint64)e[i--] << (11 - c); + c += 53; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_4096_get_from_table_39(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 11; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 49; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 11; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 53 - c; } - y = (int)((n >> 52) & 1); - n = (sp_uint64)n << 1; - - sp_4096_mont_mul_39(t[y^1], t[0], t[1], m, mp); + sp_4096_mont_sqr_39(rt, rt, m, mp); + sp_4096_mont_sqr_39(rt, rt, m, mp); + sp_4096_mont_sqr_39(rt, rt, m, mp); + sp_4096_mont_sqr_39(rt, rt, m, mp); - XMEMCPY(t[2], (void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), - sizeof(*t[2]) * 39 * 2); - sp_4096_mont_sqr_39(t[2], t[2], m, mp); - XMEMCPY((void*)(((size_t)t[0] & addr_mask[y^1]) + - ((size_t)t[1] & addr_mask[y])), t[2], - sizeof(*t[2]) * 39 * 2); + sp_4096_get_from_table_39(ct, t, y); + sp_4096_mont_mul_39(rt, rt, ct, m, mp); } - sp_4096_mont_reduce_39(t[0], m, mp); - n = sp_4096_cmp_39(t[0], m); - sp_4096_cond_sub_39(t[0], t[0], m, (sp_digit)~(n >> 63)); - XMEMCPY(r, t[0], sizeof(*r) * 39 * 2); + sp_4096_mont_reduce_39(rt, m, mp); + n = sp_4096_cmp_39(rt, m); + sp_4096_cond_sub_39(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 78); } SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#endif #else SP_DECL_VAR(sp_digit, td, (32 * 78) + 78); sp_digit* t[32]; @@ -20516,6 +23161,40 @@ static int sp_4096_mod_78(sp_digit* r, const sp_digit* a, const sp_digit* m) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || defined(WOLFSSL_HAVE_SP_DH) #if (defined(WOLFSSL_HAVE_SP_RSA) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)) || \ defined(WOLFSSL_HAVE_SP_DH) +#if !defined(WC_NO_HARDEN) && !defined(WC_NO_CACHE_RESISTANT) && \ + !defined(WOLFSSL_SP_LADDER_MODEXP) && \ + (!defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SP_FAST_MODEXP)) +/* Get an entry from the table by index in a cache resistant manner. + * + * @param [out] r Entry retrieved from table. + * @param [in] table Table of entries to choose from. + * @param [in] idx Index of entry to retrieve. + */ +static void sp_4096_get_from_table_78(sp_digit* r, + sp_digit** table, int idx) +{ + int e, j; + sp_uint64 mask; + sp_uint64 diff; + + for (j = 0; j < 156; j++) { + r[j] = 0; + } + + for (e = 0; e < 16; e++) { + /* mask is all-ones when e == idx and all-zeros otherwise. */ + diff = (sp_uint64)(e ^ idx); + diff = (sp_uint64)((diff | (sp_uint64)((sp_uint64)0 - diff)) >> + (64 - 1)); + mask = (sp_uint64)((sp_uint64)0 - ((sp_uint64)1 - diff)); + + for (j = 0; j < 156; j++) { + r[j] |= (sp_digit)((sp_uint64)table[e][j] & mask); + } + } +} +#endif + /* Modular exponentiate a to the e mod m. (r = a^e mod m) * * @param [out] r A single precision number that is the result of the @@ -20608,8 +23287,95 @@ static int sp_4096_mod_exp_78(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; +#elif defined(WC_NO_HARDEN) + SP_DECL_VAR(sp_digit, td, 11 * 156); + sp_digit* t[8]; + sp_digit* norm = NULL; + sp_digit* rt = NULL; + sp_digit* sq = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int j; + int l; + int y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, 11 * 156, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<8; i++) { + t[i] = td + ((i + 1) * 156); + } + rt = td + (9 * 156); + sq = td + (10 * 156); + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_78(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_78(t[0], a, m); + if (err == MP_OKAY) { + sp_4096_mul_78(t[0], t[0], norm); + err = sp_4096_mod_78(t[0], t[0], m); + } + } + else { + sp_4096_mul_78(t[0], a, norm); + err = sp_4096_mod_78(t[0], t[0], m); + } + } + + if (err == MP_OKAY) { + /* t[i] = a ^ (2i + 1) : odd powers only. */ + sp_4096_mont_sqr_78(sq, t[0], m, mp); + for (i = 1; i < 8; i++) { + sp_4096_mont_mul_78(t[i], t[i-1], sq, m, mp); + } + + /* Result starts at one in Montgomery form. */ + XMEMCPY(rt, norm, sizeof(sp_digit) * 156); + + for (i = bits - 1; i >= 0; ) { + if (((e[i / 53] >> (i % 53)) & 1) == 0) { + sp_4096_mont_sqr_78(rt, rt, m, mp); + i--; + } + else { + /* Longest window of at most 4 bits ending in a set bit. */ + l = (i >= 4) ? (i - 4 + 1) : 0; + while (((e[l / 53] >> (l % 53)) & 1) == 0) { + l++; + } + y = 0; + for (j = i; j >= l; j--) { + y = (y << 1) | (int)((e[j / 53] >> (j % 53)) & 1); + } + for (j = i - l + 1; j > 0; j--) { + sp_4096_mont_sqr_78(rt, rt, m, mp); + } + sp_4096_mont_mul_78(rt, rt, t[(y - 1) / 2], m, mp); + i = l - 1; + } + } + + sp_4096_mont_reduce_78(rt, m, mp); + n = sp_4096_cmp_78(rt, m); + sp_4096_cond_sub_78(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(*r) * 156); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return err; #elif !defined(WC_NO_CACHE_RESISTANT) +#ifdef WOLFSSL_SP_LADDER_MODEXP SP_DECL_VAR(sp_digit, td, 3 * 156); sp_digit* t[3] = {0, 0, 0}; sp_digit* norm = NULL; @@ -20684,6 +23450,125 @@ static int sp_4096_mod_exp_78(sp_digit* r, const sp_digit* a, SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); return err; +#else + SP_DECL_VAR(sp_digit, td, (16 * 156) + 312); + sp_digit* t[16]; + sp_digit* rt = NULL; + sp_digit* ct = NULL; + sp_digit* norm = NULL; + sp_digit mp = 1; + sp_digit n; + int i; + int c; + byte y; + int err = MP_OKAY; + + if (bits == 0) { + err = MP_VAL; + } + + SP_ALLOC_VAR(sp_digit, td, (16 * 156) + 312, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (err == MP_OKAY) { + norm = td; + for (i=0; i<16; i++) + t[i] = td + i * 156; + rt = td + 2496; + ct = td + 2652; + + sp_4096_mont_setup(m, &mp); + sp_4096_mont_norm_78(norm, m); + + if (reduceA != 0) { + err = sp_4096_mod_78(t[1], a, m); + if (err == MP_OKAY) { + sp_4096_mul_78(t[1], t[1], norm); + err = sp_4096_mod_78(t[1], t[1], m); + } + } + else { + sp_4096_mul_78(t[1], a, norm); + err = sp_4096_mod_78(t[1], t[1], m); + } + } + + if (err == MP_OKAY) { + sp_4096_mont_sqr_78(t[ 2], t[ 1], m, mp); + sp_4096_mont_mul_78(t[ 3], t[ 2], t[ 1], m, mp); + sp_4096_mont_sqr_78(t[ 4], t[ 2], m, mp); + sp_4096_mont_mul_78(t[ 5], t[ 3], t[ 2], m, mp); + sp_4096_mont_sqr_78(t[ 6], t[ 3], m, mp); + sp_4096_mont_mul_78(t[ 7], t[ 4], t[ 3], m, mp); + sp_4096_mont_sqr_78(t[ 8], t[ 4], m, mp); + sp_4096_mont_mul_78(t[ 9], t[ 5], t[ 4], m, mp); + sp_4096_mont_sqr_78(t[10], t[ 5], m, mp); + sp_4096_mont_mul_78(t[11], t[ 6], t[ 5], m, mp); + sp_4096_mont_sqr_78(t[12], t[ 6], m, mp); + sp_4096_mont_mul_78(t[13], t[ 7], t[ 6], m, mp); + sp_4096_mont_sqr_78(t[14], t[ 7], m, mp); + sp_4096_mont_mul_78(t[15], t[ 8], t[ 7], m, mp); + + bits = ((bits + 3) / 4) * 4; + i = ((bits + 52) / 53) - 1; + c = bits % 53; + if (c == 0) { + c = 53; + } + if (i < 78) { + n = (sp_uint64)e[i--] << (64 - c); + } + else { + n = 0; + i--; + } + if (c < 4) { + n |= (sp_uint64)e[i--] << (11 - c); + c += 53; + } + y = (int)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + sp_4096_get_from_table_78(rt, t, y); + while ((i >= 0) || (c >= 4)) { + if (c >= 4) { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c -= 4; + } + else if (c == 0) { + n = (sp_uint64)e[i--] << 11; + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)n << 4; + c = 49; + } + else { + y = (byte)((n >> 60) & 0xf); + n = (sp_uint64)e[i--] << 11; + c = 4 - c; + y |= (byte)((n >> (64 - c)) & ((1 << c) - 1)); + n = (sp_uint64)n << c; + c = 53 - c; + } + + sp_4096_mont_sqr_78(rt, rt, m, mp); + sp_4096_mont_sqr_78(rt, rt, m, mp); + sp_4096_mont_sqr_78(rt, rt, m, mp); + sp_4096_mont_sqr_78(rt, rt, m, mp); + + sp_4096_get_from_table_78(ct, t, y); + sp_4096_mont_mul_78(rt, rt, ct, m, mp); + } + + sp_4096_mont_reduce_78(rt, m, mp); + n = sp_4096_cmp_78(rt, m); + sp_4096_cond_sub_78(rt, rt, m, (sp_digit)~(n >> 63)); + XMEMCPY(r, rt, sizeof(sp_digit) * 156); + } + + SP_FREE_VAR(td, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return err; +#endif #else SP_DECL_VAR(sp_digit, td, (16 * 156) + 156); sp_digit* t[16];